diff --git a/make/hotspot/gensrc/GensrcAdlc.gmk b/make/hotspot/gensrc/GensrcAdlc.gmk
index 1005bc5000b..1239928c96c 100644
--- a/make/hotspot/gensrc/GensrcAdlc.gmk
+++ b/make/hotspot/gensrc/GensrcAdlc.gmk
@@ -172,6 +172,8 @@ ifeq ($(call check-jvm-feature, compiler2), true)
 
   ifeq ($(call check-jvm-feature, zgc), true)
     AD_SRC_FILES += $(call uniq, $(wildcard $(foreach d, $(AD_SRC_ROOTS), \
+        $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/x/x_$(HOTSPOT_TARGET_CPU).ad \
+        $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/x/x_$(HOTSPOT_TARGET_CPU_ARCH).ad \
         $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/z/z_$(HOTSPOT_TARGET_CPU).ad \
         $d/cpu/$(HOTSPOT_TARGET_CPU_ARCH)/gc/z/z_$(HOTSPOT_TARGET_CPU_ARCH).ad \
       )))
diff --git a/make/hotspot/lib/JvmFeatures.gmk b/make/hotspot/lib/JvmFeatures.gmk
index 1e24475ea46..cbe60fde205 100644
--- a/make/hotspot/lib/JvmFeatures.gmk
+++ b/make/hotspot/lib/JvmFeatures.gmk
@@ -149,6 +149,7 @@ endif
 ifneq ($(call check-jvm-feature, zgc), true)
   JVM_CFLAGS_FEATURES += -DINCLUDE_ZGC=0
   JVM_EXCLUDE_PATTERNS += gc/z
+  JVM_EXCLUDE_PATTERNS += gc/x
 endif
 
 ifneq ($(call check-jvm-feature, shenandoahgc), true)
diff --git a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
index dc19c72fd11..4bf7fee936b 100644
--- a/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/c1_LIRAssembler_aarch64.cpp
@@ -1010,7 +1010,7 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
       __ decode_heap_oop(dest->as_register());
     }
 
-    if (!UseZGC) {
+    if (!(UseZGC && !ZGenerational)) {
       // Load barrier has not yet been applied, so ZGC can't verify the oop here
       __ verify_oop(dest->as_register());
     }
diff --git a/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp
index ffccaa07b9d..5169a510154 100644
--- a/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/shared/barrierSetNMethod_aarch64.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -129,20 +129,14 @@ public:
   }
 };
 
-// Store the instruction bitmask, bits and name for checking the barrier.
-struct CheckInsn {
-  uint32_t mask;
-  uint32_t bits;
-  const char *name;
-};
-
 // The first instruction of the nmethod entry barrier is an ldr (literal)
 // instruction. Verify that it's really there, so the offsets are not skewed.
 bool NativeNMethodBarrier::check_barrier(err_msg& msg) const {
   uint32_t* addr = (uint32_t*) instruction_address();
   uint32_t inst = *addr;
   if ((inst & 0xff000000) != 0x18000000) {
-    msg.print("Addr: " INTPTR_FORMAT " Code: 0x%x not an ldr", p2i(addr), inst);
+    msg.print("Nmethod entry barrier did not start with ldr (literal) as expected. "
+              "Addr: " PTR_FORMAT " Code: " UINT32_FORMAT, p2i(addr), inst);
     return false;
   }
   return true;
diff --git a/src/hotspot/cpu/aarch64/gc/x/xBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/x/xBarrierSetAssembler_aarch64.cpp
new file mode 100644
index 00000000000..5c891e8c170
--- /dev/null
+++ b/src/hotspot/cpu/aarch64/gc/x/xBarrierSetAssembler_aarch64.cpp
@@ -0,0 +1,462 @@
+/*
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "asm/macroAssembler.inline.hpp"
+#include "code/codeBlob.hpp"
+#include "code/vmreg.inline.hpp"
+#include "gc/x/xBarrier.inline.hpp"
+#include "gc/x/xBarrierSet.hpp"
+#include "gc/x/xBarrierSetAssembler.hpp"
+#include "gc/x/xBarrierSetRuntime.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+#include "memory/resourceArea.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "utilities/macros.hpp"
+#ifdef COMPILER1
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/x/c1/xBarrierSetC1.hpp"
+#endif // COMPILER1
+#ifdef COMPILER2
+#include "gc/x/c2/xBarrierSetC2.hpp"
+#endif // COMPILER2
+
+#ifdef PRODUCT
+#define BLOCK_COMMENT(str) /* nothing */
+#else
+#define BLOCK_COMMENT(str) __ block_comment(str)
+#endif
+
+#undef __
+#define __ masm->
+
+void XBarrierSetAssembler::load_at(MacroAssembler* masm,
+                                   DecoratorSet decorators,
+                                   BasicType type,
+                                   Register dst,
+                                   Address src,
+                                   Register tmp1,
+                                   Register tmp2) {
+  if (!XBarrierSet::barrier_needed(decorators, type)) {
+    // Barrier not needed
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
+    return;
+  }
+
+  assert_different_registers(rscratch1, rscratch2, src.base());
+  assert_different_registers(rscratch1, rscratch2, dst);
+
+  Label done;
+
+  // Load bad mask into scratch register.
+  __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
+  __ lea(rscratch2, src);
+  __ ldr(dst, src);
+
+  // Test reference against bad mask. If mask bad, then we need to fix it up.
+  __ tst(dst, rscratch1);
+  __ br(Assembler::EQ, done);
+
+  __ enter(/*strip_ret_addr*/true);
+
+  __ push_call_clobbered_registers_except(RegSet::of(dst));
+
+  if (c_rarg0 != dst) {
+    __ mov(c_rarg0, dst);
+  }
+  __ mov(c_rarg1, rscratch2);
+
+  __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
+
+  // Make sure dst has the return value.
+  if (dst != r0) {
+    __ mov(dst, r0);
+  }
+
+  __ pop_call_clobbered_registers_except(RegSet::of(dst));
+  __ leave();
+
+  __ bind(done);
+}
+
+#ifdef ASSERT
+
+void XBarrierSetAssembler::store_at(MacroAssembler* masm,
+                                        DecoratorSet decorators,
+                                        BasicType type,
+                                        Address dst,
+                                        Register val,
+                                        Register tmp1,
+                                        Register tmp2,
+                                        Register tmp3) {
+  // Verify value
+  if (is_reference_type(type)) {
+    // Note that src could be noreg, which means we
+    // are storing null and can skip verification.
+    if (val != noreg) {
+      Label done;
+
+      // tmp1, tmp2 and tmp3 are often set to noreg.
+      RegSet savedRegs = RegSet::of(rscratch1);
+      __ push(savedRegs, sp);
+
+      __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
+      __ tst(val, rscratch1);
+      __ br(Assembler::EQ, done);
+      __ stop("Verify oop store failed");
+      __ should_not_reach_here();
+      __ bind(done);
+      __ pop(savedRegs, sp);
+    }
+  }
+
+  // Store value
+  BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, noreg);
+}
+
+#endif // ASSERT
+
+void XBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
+                                              DecoratorSet decorators,
+                                              bool is_oop,
+                                              Register src,
+                                              Register dst,
+                                              Register count,
+                                              RegSet saved_regs) {
+  if (!is_oop) {
+    // Barrier not needed
+    return;
+  }
+
+  BLOCK_COMMENT("XBarrierSetAssembler::arraycopy_prologue {");
+
+  assert_different_registers(src, count, rscratch1);
+
+  __ push(saved_regs, sp);
+
+  if (count == c_rarg0) {
+    if (src == c_rarg1) {
+      // exactly backwards!!
+      __ mov(rscratch1, c_rarg0);
+      __ mov(c_rarg0, c_rarg1);
+      __ mov(c_rarg1, rscratch1);
+    } else {
+      __ mov(c_rarg1, count);
+      __ mov(c_rarg0, src);
+    }
+  } else {
+    __ mov(c_rarg0, src);
+    __ mov(c_rarg1, count);
+  }
+
+  __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2);
+
+  __ pop(saved_regs, sp);
+
+  BLOCK_COMMENT("} XBarrierSetAssembler::arraycopy_prologue");
+}
+
+void XBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
+                                                         Register jni_env,
+                                                         Register robj,
+                                                         Register tmp,
+                                                         Label& slowpath) {
+  BLOCK_COMMENT("XBarrierSetAssembler::try_resolve_jobject_in_native {");
+
+  assert_different_registers(jni_env, robj, tmp);
+
+  // Resolve jobject
+  BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath);
+
+  // The Address offset is too large to direct load - -784. Our range is +127, -128.
+  __ mov(tmp, (int64_t)(in_bytes(XThreadLocalData::address_bad_mask_offset()) -
+              in_bytes(JavaThread::jni_environment_offset())));
+
+  // Load address bad mask
+  __ add(tmp, jni_env, tmp);
+  __ ldr(tmp, Address(tmp));
+
+  // Check address bad mask
+  __ tst(robj, tmp);
+  __ br(Assembler::NE, slowpath);
+
+  BLOCK_COMMENT("} XBarrierSetAssembler::try_resolve_jobject_in_native");
+}
+
+#ifdef COMPILER1
+
+#undef __
+#define __ ce->masm()->
+
+void XBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
+                                                         LIR_Opr ref) const {
+  assert_different_registers(rscratch1, rthread, ref->as_register());
+
+  __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
+  __ tst(ref->as_register(), rscratch1);
+}
+
+void XBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
+                                                         XLoadBarrierStubC1* stub) const {
+  // Stub entry
+  __ bind(*stub->entry());
+
+  Register ref = stub->ref()->as_register();
+  Register ref_addr = noreg;
+  Register tmp = noreg;
+
+  if (stub->tmp()->is_valid()) {
+    // Load address into tmp register
+    ce->leal(stub->ref_addr(), stub->tmp());
+    ref_addr = tmp = stub->tmp()->as_pointer_register();
+  } else {
+    // Address already in register
+    ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
+  }
+
+  assert_different_registers(ref, ref_addr, noreg);
+
+  // Save r0 unless it is the result or tmp register
+  // Set up SP to accommodate parameters and maybe r0..
+  if (ref != r0 && tmp != r0) {
+    __ sub(sp, sp, 32);
+    __ str(r0, Address(sp, 16));
+  } else {
+    __ sub(sp, sp, 16);
+  }
+
+  // Setup arguments and call runtime stub
+  ce->store_parameter(ref_addr, 1);
+  ce->store_parameter(ref, 0);
+
+  __ far_call(stub->runtime_stub());
+
+  // Verify result
+  __ verify_oop(r0);
+
+  // Move result into place
+  if (ref != r0) {
+    __ mov(ref, r0);
+  }
+
+  // Restore r0 unless it is the result or tmp register
+  if (ref != r0 && tmp != r0) {
+    __ ldr(r0, Address(sp, 16));
+    __ add(sp, sp, 32);
+  } else {
+    __ add(sp, sp, 16);
+  }
+
+  // Stub exit
+  __ b(*stub->continuation());
+}
+
+#undef __
+#define __ sasm->
+
+void XBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
+                                                                 DecoratorSet decorators) const {
+  __ prologue("zgc_load_barrier stub", false);
+
+  __ push_call_clobbered_registers_except(RegSet::of(r0));
+
+  // Setup arguments
+  __ load_parameter(0, c_rarg0);
+  __ load_parameter(1, c_rarg1);
+
+  __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
+
+  __ pop_call_clobbered_registers_except(RegSet::of(r0));
+
+  __ epilogue();
+}
+#endif // COMPILER1
+
+#ifdef COMPILER2
+
+OptoReg::Name XBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
+  if (!OptoReg::is_reg(opto_reg)) {
+    return OptoReg::Bad;
+  }
+
+  const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+  if (vm_reg->is_FloatRegister()) {
+    return opto_reg & ~1;
+  }
+
+  return opto_reg;
+}
+
+#undef __
+#define __ _masm->
+
+class XSaveLiveRegisters {
+private:
+  MacroAssembler* const _masm;
+  RegSet                _gp_regs;
+  FloatRegSet           _fp_regs;
+  PRegSet               _p_regs;
+
+public:
+  void initialize(XLoadBarrierStubC2* stub) {
+    // Record registers that needs to be saved/restored
+    RegMaskIterator rmi(stub->live());
+    while (rmi.has_next()) {
+      const OptoReg::Name opto_reg = rmi.next();
+      if (OptoReg::is_reg(opto_reg)) {
+        const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+        if (vm_reg->is_Register()) {
+          _gp_regs += RegSet::of(vm_reg->as_Register());
+        } else if (vm_reg->is_FloatRegister()) {
+          _fp_regs += FloatRegSet::of(vm_reg->as_FloatRegister());
+        } else if (vm_reg->is_PRegister()) {
+          _p_regs += PRegSet::of(vm_reg->as_PRegister());
+        } else {
+          fatal("Unknown register type");
+        }
+      }
+    }
+
+    // Remove C-ABI SOE registers, scratch regs and _ref register that will be updated
+    _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9, stub->ref());
+  }
+
+  XSaveLiveRegisters(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
+      _masm(masm),
+      _gp_regs(),
+      _fp_regs(),
+      _p_regs() {
+
+    // Figure out what registers to save/restore
+    initialize(stub);
+
+    // Save registers
+    __ push(_gp_regs, sp);
+    __ push_fp(_fp_regs, sp);
+    __ push_p(_p_regs, sp);
+  }
+
+  ~XSaveLiveRegisters() {
+    // Restore registers
+    __ pop_p(_p_regs, sp);
+    __ pop_fp(_fp_regs, sp);
+
+    // External runtime call may clobber ptrue reg
+    __ reinitialize_ptrue();
+
+    __ pop(_gp_regs, sp);
+  }
+};
+
+#undef __
+#define __ _masm->
+
+class XSetupArguments {
+private:
+  MacroAssembler* const _masm;
+  const Register        _ref;
+  const Address         _ref_addr;
+
+public:
+  XSetupArguments(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
+      _masm(masm),
+      _ref(stub->ref()),
+      _ref_addr(stub->ref_addr()) {
+
+    // Setup arguments
+    if (_ref_addr.base() == noreg) {
+      // No self healing
+      if (_ref != c_rarg0) {
+        __ mov(c_rarg0, _ref);
+      }
+      __ mov(c_rarg1, 0);
+    } else {
+      // Self healing
+      if (_ref == c_rarg0) {
+        // _ref is already at correct place
+        __ lea(c_rarg1, _ref_addr);
+      } else if (_ref != c_rarg1) {
+        // _ref is in wrong place, but not in c_rarg1, so fix it first
+        __ lea(c_rarg1, _ref_addr);
+        __ mov(c_rarg0, _ref);
+      } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) {
+        assert(_ref == c_rarg1, "Mov ref first, vacating c_rarg0");
+        __ mov(c_rarg0, _ref);
+        __ lea(c_rarg1, _ref_addr);
+      } else {
+        assert(_ref == c_rarg1, "Need to vacate c_rarg1 and _ref_addr is using c_rarg0");
+        if (_ref_addr.base() == c_rarg0 || _ref_addr.index() == c_rarg0) {
+          __ mov(rscratch2, c_rarg1);
+          __ lea(c_rarg1, _ref_addr);
+          __ mov(c_rarg0, rscratch2);
+        } else {
+          ShouldNotReachHere();
+        }
+      }
+    }
+  }
+
+  ~XSetupArguments() {
+    // Transfer result
+    if (_ref != r0) {
+      __ mov(_ref, r0);
+    }
+  }
+};
+
+#undef __
+#define __ masm->
+
+void XBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, XLoadBarrierStubC2* stub) const {
+  BLOCK_COMMENT("XLoadBarrierStubC2");
+
+  // Stub entry
+  __ bind(*stub->entry());
+
+  {
+    XSaveLiveRegisters save_live_registers(masm, stub);
+    XSetupArguments setup_arguments(masm, stub);
+    __ mov(rscratch1, stub->slow_path());
+    __ blr(rscratch1);
+  }
+  // Stub exit
+  __ b(*stub->continuation());
+}
+
+#endif // COMPILER2
+
+#undef __
+#define __ masm->
+
+void XBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
+  // Check if mask is good.
+  // verifies that XAddressBadMask & r0 == 0
+  __ ldr(tmp2, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
+  __ andr(tmp1, obj, tmp2);
+  __ cbnz(tmp1, error);
+
+  BarrierSetAssembler::check_oop(masm, obj, tmp1, tmp2, error);
+}
+
+#undef __
diff --git a/src/hotspot/cpu/aarch64/gc/x/xBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/x/xBarrierSetAssembler_aarch64.hpp
new file mode 100644
index 00000000000..8c1e9521757
--- /dev/null
+++ b/src/hotspot/cpu/aarch64/gc/x/xBarrierSetAssembler_aarch64.hpp
@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_AARCH64_GC_X_XBARRIERSETASSEMBLER_AARCH64_HPP
+#define CPU_AARCH64_GC_X_XBARRIERSETASSEMBLER_AARCH64_HPP
+
+#include "code/vmreg.hpp"
+#include "oops/accessDecorators.hpp"
+#ifdef COMPILER2
+#include "opto/optoreg.hpp"
+#endif // COMPILER2
+
+#ifdef COMPILER1
+class LIR_Assembler;
+class LIR_Opr;
+class StubAssembler;
+#endif // COMPILER1
+
+#ifdef COMPILER2
+class Node;
+#endif // COMPILER2
+
+#ifdef COMPILER1
+class XLoadBarrierStubC1;
+#endif // COMPILER1
+
+#ifdef COMPILER2
+class XLoadBarrierStubC2;
+#endif // COMPILER2
+
+class XBarrierSetAssembler : public XBarrierSetAssemblerBase {
+public:
+  virtual void load_at(MacroAssembler* masm,
+                       DecoratorSet decorators,
+                       BasicType type,
+                       Register dst,
+                       Address src,
+                       Register tmp1,
+                       Register tmp2);
+
+#ifdef ASSERT
+  virtual void store_at(MacroAssembler* masm,
+                        DecoratorSet decorators,
+                        BasicType type,
+                        Address dst,
+                        Register val,
+                        Register tmp1,
+                        Register tmp2,
+                        Register tmp3);
+#endif // ASSERT
+
+  virtual void arraycopy_prologue(MacroAssembler* masm,
+                                  DecoratorSet decorators,
+                                  bool is_oop,
+                                  Register src,
+                                  Register dst,
+                                  Register count,
+                                  RegSet saved_regs);
+
+  virtual void try_resolve_jobject_in_native(MacroAssembler* masm,
+                                             Register jni_env,
+                                             Register robj,
+                                             Register tmp,
+                                             Label& slowpath);
+
+  virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
+
+#ifdef COMPILER1
+  void generate_c1_load_barrier_test(LIR_Assembler* ce,
+                                     LIR_Opr ref) const;
+
+  void generate_c1_load_barrier_stub(LIR_Assembler* ce,
+                                     XLoadBarrierStubC1* stub) const;
+
+  void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
+                                             DecoratorSet decorators) const;
+#endif // COMPILER1
+
+#ifdef COMPILER2
+  OptoReg::Name refine_register(const Node* node,
+                                OptoReg::Name opto_reg);
+
+  void generate_c2_load_barrier_stub(MacroAssembler* masm,
+                                     XLoadBarrierStubC2* stub) const;
+#endif // COMPILER2
+
+  void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
+};
+
+#endif // CPU_AARCH64_GC_X_XBARRIERSETASSEMBLER_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/x/xGlobals_aarch64.cpp
similarity index 98%
rename from src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.cpp
rename to src/hotspot/cpu/aarch64/gc/x/xGlobals_aarch64.cpp
index 5ee7c2b231c..6204f212703 100644
--- a/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/x/xGlobals_aarch64.cpp
@@ -24,7 +24,7 @@
 #include "precompiled.hpp"
 #include "gc/shared/gcLogPrecious.hpp"
 #include "gc/shared/gc_globals.hpp"
-#include "gc/z/zGlobals.hpp"
+#include "gc/x/xGlobals.hpp"
 #include "runtime/globals.hpp"
 #include "runtime/os.hpp"
 #include "utilities/globalDefinitions.hpp"
@@ -196,15 +196,15 @@ static size_t probe_valid_max_address_bit() {
 #endif // LINUX
 }
 
-size_t ZPlatformAddressOffsetBits() {
+size_t XPlatformAddressOffsetBits() {
   const static size_t valid_max_address_offset_bits = probe_valid_max_address_bit() + 1;
   const size_t max_address_offset_bits = valid_max_address_offset_bits - 3;
   const size_t min_address_offset_bits = max_address_offset_bits - 2;
-  const size_t address_offset = round_up_power_of_2(MaxHeapSize * ZVirtualToPhysicalRatio);
+  const size_t address_offset = round_up_power_of_2(MaxHeapSize * XVirtualToPhysicalRatio);
   const size_t address_offset_bits = log2i_exact(address_offset);
   return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
 }
 
-size_t ZPlatformAddressMetadataShift() {
-  return ZPlatformAddressOffsetBits();
+size_t XPlatformAddressMetadataShift() {
+  return XPlatformAddressOffsetBits();
 }
diff --git a/src/hotspot/cpu/aarch64/gc/x/xGlobals_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/x/xGlobals_aarch64.hpp
new file mode 100644
index 00000000000..870b0d74d57
--- /dev/null
+++ b/src/hotspot/cpu/aarch64/gc/x/xGlobals_aarch64.hpp
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_AARCH64_GC_X_XGLOBALS_AARCH64_HPP
+#define CPU_AARCH64_GC_X_XGLOBALS_AARCH64_HPP
+
+const size_t XPlatformHeapViews        = 3;
+const size_t XPlatformCacheLineSize    = 64;
+
+size_t XPlatformAddressOffsetBits();
+size_t XPlatformAddressMetadataShift();
+
+#endif // CPU_AARCH64_GC_X_XGLOBALS_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/gc/x/x_aarch64.ad b/src/hotspot/cpu/aarch64/gc/x/x_aarch64.ad
new file mode 100644
index 00000000000..a8ef3ce9f13
--- /dev/null
+++ b/src/hotspot/cpu/aarch64/gc/x/x_aarch64.ad
@@ -0,0 +1,243 @@
+//
+// Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+
+source_hpp %{
+
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/c2/xBarrierSetC2.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+
+%}
+
+source %{
+
+static void x_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data) {
+  if (barrier_data == XLoadBarrierElided) {
+    return;
+  }
+  XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
+  __ ldr(tmp, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
+  __ andr(tmp, tmp, ref);
+  __ cbnz(tmp, *stub->entry());
+  __ bind(*stub->continuation());
+}
+
+static void x_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
+  XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, XLoadBarrierStrong);
+  __ b(*stub->entry());
+  __ bind(*stub->continuation());
+}
+
+%}
+
+// Load Pointer
+instruct xLoadP(iRegPNoSp dst, memory8 mem, rFlagsReg cr)
+%{
+  match(Set dst (LoadP mem));
+  predicate(UseZGC && !ZGenerational && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() != 0));
+  effect(TEMP dst, KILL cr);
+
+  ins_cost(4 * INSN_COST);
+
+  format %{ "ldr  $dst, $mem" %}
+
+  ins_encode %{
+    const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
+    __ ldr($dst$$Register, ref_addr);
+    x_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, barrier_data());
+  %}
+
+  ins_pipe(iload_reg_mem);
+%}
+
+// Load Pointer Volatile
+instruct xLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr)
+%{
+  match(Set dst (LoadP mem));
+  predicate(UseZGC && !ZGenerational && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
+  effect(TEMP dst, KILL cr);
+
+  ins_cost(VOLATILE_REF_COST);
+
+  format %{ "ldar  $dst, $mem\t" %}
+
+  ins_encode %{
+    __ ldar($dst$$Register, $mem$$Register);
+    x_load_barrier(_masm, this, Address($mem$$Register), $dst$$Register, rscratch2 /* tmp */, barrier_data());
+  %}
+
+  ins_pipe(pipe_serial);
+%}
+
+instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+  predicate(UseZGC && !ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
+  effect(KILL cr, TEMP_DEF res);
+
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  format %{ "cmpxchg $mem, $oldval, $newval\n\t"
+            "cset    $res, EQ" %}
+
+  ins_encode %{
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+               false /* acquire */, true /* release */, false /* weak */, rscratch2);
+    __ cset($res$$Register, Assembler::EQ);
+    if (barrier_data() != XLoadBarrierElided) {
+      Label good;
+      __ ldr(rscratch1, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
+      __ andr(rscratch1, rscratch1, rscratch2);
+      __ cbz(rscratch1, good);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */);
+      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+                 false /* acquire */, true /* release */, false /* weak */, rscratch2);
+      __ cset($res$$Register, Assembler::EQ);
+      __ bind(good);
+    }
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+  predicate(UseZGC && !ZGenerational && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == XLoadBarrierStrong));
+  effect(KILL cr, TEMP_DEF res);
+
+  ins_cost(2 * VOLATILE_REF_COST);
+
+ format %{ "cmpxchg $mem, $oldval, $newval\n\t"
+           "cset    $res, EQ" %}
+
+  ins_encode %{
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+               true /* acquire */, true /* release */, false /* weak */, rscratch2);
+    __ cset($res$$Register, Assembler::EQ);
+    if (barrier_data() != XLoadBarrierElided) {
+      Label good;
+      __ ldr(rscratch1, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
+      __ andr(rscratch1, rscratch1, rscratch2);
+      __ cbz(rscratch1, good);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */ );
+      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+                 true /* acquire */, true /* release */, false /* weak */, rscratch2);
+      __ cset($res$$Register, Assembler::EQ);
+      __ bind(good);
+    }
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+  match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+  predicate(UseZGC && !ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
+  effect(TEMP_DEF res, KILL cr);
+
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  format %{ "cmpxchg $res = $mem, $oldval, $newval" %}
+
+  ins_encode %{
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+               false /* acquire */, true /* release */, false /* weak */, $res$$Register);
+    if (barrier_data() != XLoadBarrierElided) {
+      Label good;
+      __ ldr(rscratch1, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
+      __ andr(rscratch1, rscratch1, $res$$Register);
+      __ cbz(rscratch1, good);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */);
+      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+                 false /* acquire */, true /* release */, false /* weak */, $res$$Register);
+      __ bind(good);
+    }
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+  match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+  predicate(UseZGC && !ZGenerational && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
+  effect(TEMP_DEF res, KILL cr);
+
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  format %{ "cmpxchg $res = $mem, $oldval, $newval" %}
+
+  ins_encode %{
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+               true /* acquire */, true /* release */, false /* weak */, $res$$Register);
+    if (barrier_data() != XLoadBarrierElided) {
+      Label good;
+      __ ldr(rscratch1, Address(rthread, XThreadLocalData::address_bad_mask_offset()));
+      __ andr(rscratch1, rscratch1, $res$$Register);
+      __ cbz(rscratch1, good);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */);
+      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+                 true /* acquire */, true /* release */, false /* weak */, $res$$Register);
+      __ bind(good);
+    }
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct xGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+  match(Set prev (GetAndSetP mem newv));
+  predicate(UseZGC && !ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP_DEF prev, KILL cr);
+
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
+
+  ins_encode %{
+    __ atomic_xchg($prev$$Register, $newv$$Register, $mem$$Register);
+    x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, barrier_data());
+  %}
+
+  ins_pipe(pipe_serial);
+%}
+
+instruct xGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+  match(Set prev (GetAndSetP mem newv));
+  predicate(UseZGC && !ZGenerational && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() != 0));
+  effect(TEMP_DEF prev, KILL cr);
+
+  ins_cost(VOLATILE_REF_COST);
+
+  format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
+
+  ins_encode %{
+    __ atomic_xchgal($prev$$Register, $newv$$Register, $mem$$Register);
+    x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, barrier_data());
+  %}
+  ins_pipe(pipe_serial);
+%}
diff --git a/src/hotspot/cpu/aarch64/gc/z/zAddress_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zAddress_aarch64.cpp
new file mode 100644
index 00000000000..6c3cea73d1a
--- /dev/null
+++ b/src/hotspot/cpu/aarch64/gc/z/zAddress_aarch64.cpp
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/z/zAddress.hpp"
+#include "gc/z/zBarrierSetAssembler.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/os.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+#ifdef LINUX
+#include <sys/mman.h>
+#endif // LINUX
+
+// Default value if probing is not implemented for a certain platform: 128TB
+static const size_t DEFAULT_MAX_ADDRESS_BIT = 47;
+// Minimum value returned, if probing fails: 64GB
+static const size_t MINIMUM_MAX_ADDRESS_BIT = 36;
+
+static size_t probe_valid_max_address_bit() {
+#ifdef LINUX
+  size_t max_address_bit = 0;
+  const size_t page_size = os::vm_page_size();
+  for (size_t i = DEFAULT_MAX_ADDRESS_BIT; i > MINIMUM_MAX_ADDRESS_BIT; --i) {
+    const uintptr_t base_addr = ((uintptr_t) 1U) << i;
+    if (msync((void*)base_addr, page_size, MS_ASYNC) == 0) {
+      // msync suceeded, the address is valid, and maybe even already mapped.
+      max_address_bit = i;
+      break;
+    }
+    if (errno != ENOMEM) {
+      // Some error occured. This should never happen, but msync
+      // has some undefined behavior, hence ignore this bit.
+#ifdef ASSERT
+      fatal("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
+#else // ASSERT
+      log_warning_p(gc)("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
+#endif // ASSERT
+      continue;
+    }
+    // Since msync failed with ENOMEM, the page might not be mapped.
+    // Try to map it, to see if the address is valid.
+    void* const result_addr = mmap((void*) base_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
+    if (result_addr != MAP_FAILED) {
+      munmap(result_addr, page_size);
+    }
+    if ((uintptr_t) result_addr == base_addr) {
+      // address is valid
+      max_address_bit = i;
+      break;
+    }
+  }
+  if (max_address_bit == 0) {
+    // probing failed, allocate a very high page and take that bit as the maximum
+    const uintptr_t high_addr = ((uintptr_t) 1U) << DEFAULT_MAX_ADDRESS_BIT;
+    void* const result_addr = mmap((void*) high_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
+    if (result_addr != MAP_FAILED) {
+      max_address_bit = BitsPerSize_t - count_leading_zeros((size_t) result_addr) - 1;
+      munmap(result_addr, page_size);
+    }
+  }
+  log_info_p(gc, init)("Probing address space for the highest valid bit: " SIZE_FORMAT, max_address_bit);
+  return MAX2(max_address_bit, MINIMUM_MAX_ADDRESS_BIT);
+#else // LINUX
+  return DEFAULT_MAX_ADDRESS_BIT;
+#endif // LINUX
+}
+
+size_t ZPlatformAddressOffsetBits() {
+  const static size_t valid_max_address_offset_bits = probe_valid_max_address_bit() + 1;
+  const size_t max_address_offset_bits = valid_max_address_offset_bits - 3;
+  const size_t min_address_offset_bits = max_address_offset_bits - 2;
+  const size_t address_offset = round_up_power_of_2(MaxHeapSize * ZVirtualToPhysicalRatio);
+  const size_t address_offset_bits = log2i_exact(address_offset);
+  return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
+}
+
+size_t ZPlatformAddressHeapBaseShift() {
+  return ZPlatformAddressOffsetBits();
+}
+
+void ZGlobalsPointers::pd_set_good_masks() {
+  BarrierSetAssembler::clear_patching_epoch();
+}
diff --git a/src/hotspot/cpu/aarch64/gc/z/zAddress_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zAddress_aarch64.hpp
new file mode 100644
index 00000000000..430ee53ebfb
--- /dev/null
+++ b/src/hotspot/cpu/aarch64/gc/z/zAddress_aarch64.hpp
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_AARCH64_GC_Z_ZADDRESS_AARCH64_HPP
+#define CPU_AARCH64_GC_Z_ZADDRESS_AARCH64_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+const size_t ZPointerLoadShift = 16;
+
+size_t ZPlatformAddressOffsetBits();
+size_t ZPlatformAddressHeapBaseShift();
+
+#endif // CPU_AARCH64_GC_Z_ZADDRESS_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/gc/z/zAddress_aarch64.inline.hpp b/src/hotspot/cpu/aarch64/gc/z/zAddress_aarch64.inline.hpp
new file mode 100644
index 00000000000..1102254a037
--- /dev/null
+++ b/src/hotspot/cpu/aarch64/gc/z/zAddress_aarch64.inline.hpp
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_AARCH64_GC_Z_ZADDRESS_AARCH64_INLINE_HPP
+#define CPU_AARCH64_GC_Z_ZADDRESS_AARCH64_INLINE_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+inline uintptr_t ZPointer::remap_bits(uintptr_t colored) {
+  return (colored ^ ZPointerRemappedMask) & ZPointerRemappedMask;
+}
+
+inline constexpr int ZPointer::load_shift_lookup(uintptr_t value) {
+  return ZPointerLoadShift;
+}
+
+#endif // CPU_AARCH64_GC_Z_ZADDRESS_AARCH64_INLINE_HPP
diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
index 6f4d201a27a..2f5f356337d 100644
--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
+++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.cpp
@@ -25,12 +25,16 @@
 #include "asm/macroAssembler.inline.hpp"
 #include "code/codeBlob.hpp"
 #include "code/vmreg.inline.hpp"
+#include "gc/z/zAddress.hpp"
 #include "gc/z/zBarrier.inline.hpp"
 #include "gc/z/zBarrierSet.hpp"
 #include "gc/z/zBarrierSetAssembler.hpp"
 #include "gc/z/zBarrierSetRuntime.hpp"
 #include "gc/z/zThreadLocalData.hpp"
 #include "memory/resourceArea.hpp"
+#include "nativeInst_aarch64.hpp"
+#include "runtime/icache.hpp"
+#include "runtime/jniHandles.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "utilities/macros.hpp"
 #ifdef COMPILER1
@@ -40,6 +44,7 @@
 #endif // COMPILER1
 #ifdef COMPILER2
 #include "gc/z/c2/zBarrierSetC2.hpp"
+#include "opto/output.hpp"
 #endif // COMPILER2
 
 #ifdef PRODUCT
@@ -51,6 +56,52 @@
 #undef __
 #define __ masm->
 
+// Helper for saving and restoring registers across a runtime call that does
+// not have any live vector registers.
+class ZRuntimeCallSpill {
+private:
+  MacroAssembler* _masm;
+  Register _result;
+
+  void save() {
+    MacroAssembler* masm = _masm;
+
+    __ enter(true /* strip_ret_addr */);
+    if (_result != noreg) {
+      __ push_call_clobbered_registers_except(RegSet::of(_result));
+    } else {
+      __ push_call_clobbered_registers();
+    }
+  }
+
+  void restore() {
+    MacroAssembler* masm = _masm;
+
+    if (_result != noreg) {
+      // Make sure _result has the return value.
+      if (_result != r0) {
+        __ mov(_result, r0);
+      }
+
+      __ pop_call_clobbered_registers_except(RegSet::of(_result));
+    } else {
+      __ pop_call_clobbered_registers();
+    }
+    __ leave();
+  }
+
+public:
+  ZRuntimeCallSpill(MacroAssembler* masm, Register result)
+    : _masm(masm),
+      _result(result) {
+    save();
+  }
+
+  ~ZRuntimeCallSpill() {
+    restore();
+  }
+};
+
 void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
                                    DecoratorSet decorators,
                                    BasicType type,
@@ -64,78 +115,329 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
     return;
   }
 
-  assert_different_registers(rscratch1, rscratch2, src.base());
-  assert_different_registers(rscratch1, rscratch2, dst);
+  assert_different_registers(tmp1, tmp2, src.base(), noreg);
+  assert_different_registers(tmp1, tmp2, src.index());
+  assert_different_registers(tmp1, tmp2, dst, noreg);
+  assert_different_registers(tmp2, rscratch1);
 
   Label done;
+  Label uncolor;
 
   // Load bad mask into scratch register.
-  __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
-  __ lea(rscratch2, src);
-  __ ldr(dst, src);
+  const bool on_non_strong =
+    (decorators & ON_WEAK_OOP_REF) != 0 ||
+    (decorators & ON_PHANTOM_OOP_REF) != 0;
+
+  if (on_non_strong) {
+    __ ldr(tmp1, mark_bad_mask_from_thread(rthread));
+  } else {
+    __ ldr(tmp1, load_bad_mask_from_thread(rthread));
+  }
+
+  __ lea(tmp2, src);
+  __ ldr(dst, tmp2);
 
   // Test reference against bad mask. If mask bad, then we need to fix it up.
-  __ tst(dst, rscratch1);
-  __ br(Assembler::EQ, done);
+  __ tst(dst, tmp1);
+  __ br(Assembler::EQ, uncolor);
 
-  __ enter(/*strip_ret_addr*/true);
+  {
+    // Call VM
+    ZRuntimeCallSpill rcs(masm, dst);
 
-  __ push_call_clobbered_registers_except(RegSet::of(dst));
+    if (c_rarg0 != dst) {
+      __ mov(c_rarg0, dst);
+    }
+    __ mov(c_rarg1, tmp2);
 
-  if (c_rarg0 != dst) {
-    __ mov(c_rarg0, dst);
-  }
-  __ mov(c_rarg1, rscratch2);
-
-  __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
-
-  // Make sure dst has the return value.
-  if (dst != r0) {
-    __ mov(dst, r0);
+    __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
   }
 
-  __ pop_call_clobbered_registers_except(RegSet::of(dst));
-  __ leave();
+  // Slow-path has already uncolored
+  __ b(done);
+
+  __ bind(uncolor);
+
+  // Remove the color bits
+  __ lsr(dst, dst, ZPointerLoadShift);
 
   __ bind(done);
 }
 
-#ifdef ASSERT
+void ZBarrierSetAssembler::store_barrier_fast(MacroAssembler* masm,
+                                              Address ref_addr,
+                                              Register rnew_zaddress,
+                                              Register rnew_zpointer,
+                                              Register rtmp,
+                                              bool in_nmethod,
+                                              bool is_atomic,
+                                              Label& medium_path,
+                                              Label& medium_path_continuation) const {
+  assert_different_registers(ref_addr.base(), rnew_zpointer, rtmp);
+  assert_different_registers(ref_addr.index(), rnew_zpointer, rtmp);
+  assert_different_registers(rnew_zaddress, rnew_zpointer, rtmp);
+
+  if (in_nmethod) {
+    if (is_atomic) {
+      __ ldrh(rtmp, ref_addr);
+      // Atomic operations must ensure that the contents of memory are store-good before
+      // an atomic operation can execute.
+      // A not relocatable object could have spurious raw null pointers in its fields after
+      // getting promoted to the old generation.
+      __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov);
+      __ movzw(rnew_zpointer, barrier_Relocation::unpatched);
+      __ cmpw(rtmp, rnew_zpointer);
+    } else {
+      __ ldr(rtmp, ref_addr);
+      // Stores on relocatable objects never need to deal with raw null pointers in fields.
+      // Raw null pointers may only exist in the young generation, as they get pruned when
+      // the object is relocated to old. And no pre-write barrier needs to perform any action
+      // in the young generation.
+      __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreBadBeforeMov);
+      __ movzw(rnew_zpointer, barrier_Relocation::unpatched);
+      __ tst(rtmp, rnew_zpointer);
+    }
+    __ br(Assembler::NE, medium_path);
+    __ bind(medium_path_continuation);
+    assert_different_registers(rnew_zaddress, rnew_zpointer);
+    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov);
+    __ movzw(rnew_zpointer, barrier_Relocation::unpatched);
+    __ orr(rnew_zpointer, rnew_zpointer, rnew_zaddress, Assembler::LSL, ZPointerLoadShift);
+  } else {
+    assert(!is_atomic, "atomics outside of nmethods not supported");
+    __ lea(rtmp, ref_addr);
+    __ ldr(rtmp, rtmp);
+    __ ldr(rnew_zpointer, Address(rthread, ZThreadLocalData::store_bad_mask_offset()));
+    __ tst(rtmp, rnew_zpointer);
+    __ br(Assembler::NE, medium_path);
+    __ bind(medium_path_continuation);
+    if (rnew_zaddress == noreg) {
+      __ eor(rnew_zpointer, rnew_zpointer, rnew_zpointer);
+    } else {
+      __ mov(rnew_zpointer, rnew_zaddress);
+    }
+
+    // Load the current good shift, and add the color bits
+    __ lsl(rnew_zpointer, rnew_zpointer, ZPointerLoadShift);
+    __ ldr(rtmp, Address(rthread, ZThreadLocalData::store_good_mask_offset()));
+    __ orr(rnew_zpointer, rnew_zpointer, rtmp);
+  }
+}
+
+static void store_barrier_buffer_add(MacroAssembler* masm,
+                                     Address ref_addr,
+                                     Register tmp1,
+                                     Register tmp2,
+                                     Label& slow_path) {
+  Address buffer(rthread, ZThreadLocalData::store_barrier_buffer_offset());
+  assert_different_registers(ref_addr.base(), ref_addr.index(), tmp1, tmp2);
+
+  __ ldr(tmp1, buffer);
+
+  // Combined pointer bump and check if the buffer is disabled or full
+  __ ldr(tmp2, Address(tmp1, ZStoreBarrierBuffer::current_offset()));
+  __ cmp(tmp2, (uint8_t)0);
+  __ br(Assembler::EQ, slow_path);
+
+  // Bump the pointer
+  __ sub(tmp2, tmp2, sizeof(ZStoreBarrierEntry));
+  __ str(tmp2, Address(tmp1, ZStoreBarrierBuffer::current_offset()));
+
+  // Compute the buffer entry address
+  __ lea(tmp2, Address(tmp2, ZStoreBarrierBuffer::buffer_offset()));
+  __ add(tmp2, tmp2, tmp1);
+
+  // Compute and log the store address
+  __ lea(tmp1, ref_addr);
+  __ str(tmp1, Address(tmp2, in_bytes(ZStoreBarrierEntry::p_offset())));
+
+  // Load and log the prev value
+  __ ldr(tmp1, tmp1);
+  __ str(tmp1, Address(tmp2, in_bytes(ZStoreBarrierEntry::prev_offset())));
+}
+
+void ZBarrierSetAssembler::store_barrier_medium(MacroAssembler* masm,
+                                                Address ref_addr,
+                                                Register rtmp1,
+                                                Register rtmp2,
+                                                Register rtmp3,
+                                                bool is_native,
+                                                bool is_atomic,
+                                                Label& medium_path_continuation,
+                                                Label& slow_path,
+                                                Label& slow_path_continuation) const {
+  assert_different_registers(ref_addr.base(), ref_addr.index(), rtmp1, rtmp2);
+
+  // The reason to end up in the medium path is that the pre-value was not 'good'.
+
+  if (is_native) {
+    __ b(slow_path);
+    __ bind(slow_path_continuation);
+    __ b(medium_path_continuation);
+  } else if (is_atomic) {
+    // Atomic accesses can get to the medium fast path because the value was a
+    // raw null value. If it was not null, then there is no doubt we need to take a slow path.
+    __ lea(rtmp2, ref_addr);
+    __ ldr(rtmp1, rtmp2);
+    __ cbnz(rtmp1, slow_path);
+
+    // If we get this far, we know there is a young raw null value in the field.
+    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov);
+    __ movzw(rtmp1, barrier_Relocation::unpatched);
+    __ cmpxchg(rtmp2, zr, rtmp1,
+               Assembler::xword,
+               false /* acquire */, false /* release */, true /* weak */,
+               rtmp3);
+    __ br(Assembler::NE, slow_path);
+
+    __ bind(slow_path_continuation);
+    __ b(medium_path_continuation);
+  } else {
+    // A non-atomic relocatable object won't get to the medium fast path due to a
+    // raw null in the young generation. We only get here because the field is bad.
+    // In this path we don't need any self healing, so we can avoid a runtime call
+    // most of the time by buffering the store barrier to be applied lazily.
+    store_barrier_buffer_add(masm,
+                             ref_addr,
+                             rtmp1,
+                             rtmp2,
+                             slow_path);
+    __ bind(slow_path_continuation);
+    __ b(medium_path_continuation);
+  }
+}
 
 void ZBarrierSetAssembler::store_at(MacroAssembler* masm,
-                                        DecoratorSet decorators,
-                                        BasicType type,
-                                        Address dst,
-                                        Register val,
-                                        Register tmp1,
-                                        Register tmp2,
-                                        Register tmp3) {
-  // Verify value
-  if (is_reference_type(type)) {
-    // Note that src could be noreg, which means we
-    // are storing null and can skip verification.
-    if (val != noreg) {
-      Label done;
+                                    DecoratorSet decorators,
+                                    BasicType type,
+                                    Address dst,
+                                    Register val,
+                                    Register tmp1,
+                                    Register tmp2,
+                                    Register tmp3) {
+  if (!ZBarrierSet::barrier_needed(decorators, type)) {
+    BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
+    return;
+  }
 
-      // tmp1, tmp2 and tmp3 are often set to noreg.
-      RegSet savedRegs = RegSet::of(rscratch1);
-      __ push(savedRegs, sp);
+  bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 
-      __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
-      __ tst(val, rscratch1);
-      __ br(Assembler::EQ, done);
-      __ stop("Verify oop store failed");
-      __ should_not_reach_here();
-      __ bind(done);
-      __ pop(savedRegs, sp);
+  assert_different_registers(val, tmp1, dst.base(), dst.index());
+
+  if (dest_uninitialized) {
+    if (val == noreg) {
+      __ eor(tmp1, tmp1, tmp1);
+    } else {
+      __ mov(tmp1, val);
     }
+    // Add the color bits
+    __ lsl(tmp1, tmp1, ZPointerLoadShift);
+    __ ldr(tmp2, Address(rthread, ZThreadLocalData::store_good_mask_offset()));
+    __ orr(tmp1, tmp2, tmp1);
+  } else {
+    Label done;
+    Label medium;
+    Label medium_continuation;
+    Label slow;
+    Label slow_continuation;
+    store_barrier_fast(masm, dst, val, tmp1, tmp2, false, false, medium, medium_continuation);
+    __ b(done);
+    __ bind(medium);
+    store_barrier_medium(masm,
+                         dst,
+                         tmp1,
+                         tmp2,
+                         noreg /* tmp3 */,
+                         false /* is_native */,
+                         false /* is_atomic */,
+                         medium_continuation,
+                         slow,
+                         slow_continuation);
+
+    __ bind(slow);
+    {
+      // Call VM
+      ZRuntimeCallSpill rcs(masm, noreg);
+      __ lea(c_rarg0, dst);
+      __ MacroAssembler::call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), 1);
+    }
+
+    __ b(slow_continuation);
+    __ bind(done);
   }
 
   // Store value
-  BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, noreg);
+  BarrierSetAssembler::store_at(masm, decorators, type, dst, tmp1, tmp2, tmp3, noreg);
 }
 
-#endif // ASSERT
+static FloatRegister z_copy_load_bad_vreg = v17;
+static FloatRegister z_copy_store_good_vreg = v18;
+static FloatRegister z_copy_store_bad_vreg = v19;
+
+static void load_wide_arraycopy_masks(MacroAssembler* masm) {
+  __ lea(rscratch1, ExternalAddress((address)&ZPointerVectorLoadBadMask));
+  __ ldrq(z_copy_load_bad_vreg, Address(rscratch1, 0));
+  __ lea(rscratch1, ExternalAddress((address)&ZPointerVectorStoreBadMask));
+  __ ldrq(z_copy_store_bad_vreg, Address(rscratch1, 0));
+  __ lea(rscratch1, ExternalAddress((address)&ZPointerVectorStoreGoodMask));
+  __ ldrq(z_copy_store_good_vreg, Address(rscratch1, 0));
+}
+
+class ZCopyRuntimeCallSpill {
+private:
+  MacroAssembler* _masm;
+  Register _result;
+
+  void save() {
+    MacroAssembler* masm = _masm;
+
+    __ enter(true /* strip_ret_addr */);
+    if (_result != noreg) {
+      __ push(__ call_clobbered_gp_registers() - RegSet::of(_result), sp);
+    } else {
+      __ push(__ call_clobbered_gp_registers(), sp);
+    }
+    int neonSize = wordSize * 2;
+    __ sub(sp, sp, 4 * neonSize);
+    __ st1(v0, v1, v2, v3, Assembler::T16B, Address(sp, 0));
+    __ sub(sp, sp, 4 * neonSize);
+    __ st1(v4, v5, v6, v7, Assembler::T16B, Address(sp, 0));
+    __ sub(sp, sp, 4 * neonSize);
+    __ st1(v16, v17, v18, v19, Assembler::T16B, Address(sp, 0));
+  }
+
+  void restore() {
+    MacroAssembler* masm = _masm;
+
+    int neonSize = wordSize * 2;
+    __ ld1(v16, v17, v18, v19, Assembler::T16B, Address(sp, 0));
+    __ add(sp, sp, 4 * neonSize);
+    __ ld1(v4, v5, v6, v7, Assembler::T16B, Address(sp, 0));
+    __ add(sp, sp, 4 * neonSize);
+    __ ld1(v0, v1, v2, v3, Assembler::T16B, Address(sp, 0));
+    __ add(sp, sp, 4 * neonSize);
+    if (_result != noreg) {
+      if (_result != r0) {
+        __ mov(_result, r0);
+      }
+      __ pop(__ call_clobbered_gp_registers() - RegSet::of(_result), sp);
+    } else {
+      __ pop(__ call_clobbered_gp_registers(), sp);
+    }
+    __ leave();
+  }
+
+public:
+  ZCopyRuntimeCallSpill(MacroAssembler* masm, Register result)
+    : _masm(masm),
+      _result(result) {
+    save();
+  }
+
+  ~ZCopyRuntimeCallSpill() {
+    restore();
+  }
+};
 
 void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
                                               DecoratorSet decorators,
@@ -151,32 +453,340 @@ void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
 
   BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {");
 
-  assert_different_registers(src, count, rscratch1);
-
-  __ push(saved_regs, sp);
-
-  if (count == c_rarg0) {
-    if (src == c_rarg1) {
-      // exactly backwards!!
-      __ mov(rscratch1, c_rarg0);
-      __ mov(c_rarg0, c_rarg1);
-      __ mov(c_rarg1, rscratch1);
-    } else {
-      __ mov(c_rarg1, count);
-      __ mov(c_rarg0, src);
-    }
-  } else {
-    __ mov(c_rarg0, src);
-    __ mov(c_rarg1, count);
-  }
-
-  __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2);
-
-  __ pop(saved_regs, sp);
+  load_wide_arraycopy_masks(masm);
 
   BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue");
 }
 
+static void copy_load_barrier(MacroAssembler* masm,
+                              Register ref,
+                              Address src,
+                              Register tmp) {
+  Label done;
+
+  __ ldr(tmp, Address(rthread, ZThreadLocalData::load_bad_mask_offset()));
+
+  // Test reference against bad mask. If mask bad, then we need to fix it up.
+  __ tst(ref, tmp);
+  __ br(Assembler::EQ, done);
+
+  {
+    // Call VM
+    ZCopyRuntimeCallSpill rcs(masm, ref);
+
+    __ lea(c_rarg1, src);
+
+    if (c_rarg0 != ref) {
+      __ mov(c_rarg0, ref);
+    }
+
+    __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(IN_HEAP | ON_STRONG_OOP_REF), 2);
+  }
+
+  // Slow-path has uncolored; revert
+  __ lsl(ref, ref, ZPointerLoadShift);
+
+  __ bind(done);
+}
+
+static void copy_load_barrier(MacroAssembler* masm,
+                              FloatRegister ref,
+                              Address src,
+                              Register tmp1,
+                              Register tmp2,
+                              FloatRegister vec_tmp) {
+  Label done;
+
+  // Test reference against bad mask. If mask bad, then we need to fix it up.
+  __ andr(vec_tmp, Assembler::T16B, ref, z_copy_load_bad_vreg);
+  __ umaxv(vec_tmp, Assembler::T16B, vec_tmp);
+  __ fcmpd(vec_tmp, 0.0);
+  __ br(Assembler::EQ, done);
+
+  __ umov(tmp2, ref, Assembler::D, 0);
+  copy_load_barrier(masm, tmp2, Address(src.base(), src.offset() + 0), tmp1);
+  __ mov(ref, __ D, 0, tmp2);
+
+  __ umov(tmp2, ref, Assembler::D, 1);
+  copy_load_barrier(masm, tmp2, Address(src.base(), src.offset() + 8), tmp1);
+  __ mov(ref, __ D, 1, tmp2);
+
+  __ bind(done);
+}
+
+static void copy_store_barrier(MacroAssembler* masm,
+                               Register pre_ref,
+                               Register new_ref,
+                               Address src,
+                               Register tmp1,
+                               Register tmp2) {
+  Label done;
+  Label slow;
+
+  // Test reference against bad mask. If mask bad, then we need to fix it up.
+  __ ldr(tmp1, Address(rthread, ZThreadLocalData::store_bad_mask_offset()));
+  __ tst(pre_ref, tmp1);
+  __ br(Assembler::EQ, done);
+
+  store_barrier_buffer_add(masm, src, tmp1, tmp2, slow);
+  __ b(done);
+
+  __ bind(slow);
+  {
+    // Call VM
+    ZCopyRuntimeCallSpill rcs(masm, noreg);
+
+    __ lea(c_rarg0, src);
+
+    __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), 1);
+  }
+
+  __ bind(done);
+
+  if (new_ref != noreg) {
+    // Set store-good color, replacing whatever color was there before
+    __ ldr(tmp1, Address(rthread, ZThreadLocalData::store_good_mask_offset()));
+    __ bfi(new_ref, tmp1, 0, 16);
+  }
+}
+
+static void copy_store_barrier(MacroAssembler* masm,
+                               FloatRegister pre_ref,
+                               FloatRegister new_ref,
+                               Address src,
+                               Register tmp1,
+                               Register tmp2,
+                               Register tmp3,
+                               FloatRegister vec_tmp) {
+  Label done;
+
+  // Test reference against bad mask. If mask bad, then we need to fix it up.
+  __ andr(vec_tmp, Assembler::T16B, pre_ref, z_copy_store_bad_vreg);
+  __ umaxv(vec_tmp, Assembler::T16B, vec_tmp);
+  __ fcmpd(vec_tmp, 0.0);
+  __ br(Assembler::EQ, done);
+
+  // Extract the 2 oops from the pre_ref vector register
+  __ umov(tmp2, pre_ref, Assembler::D, 0);
+  copy_store_barrier(masm, tmp2, noreg, Address(src.base(), src.offset() + 0), tmp1, tmp3);
+
+  __ umov(tmp2, pre_ref, Assembler::D, 1);
+  copy_store_barrier(masm, tmp2, noreg, Address(src.base(), src.offset() + 8), tmp1, tmp3);
+
+  __ bind(done);
+
+  // Remove any bad colors
+  __ bic(new_ref, Assembler::T16B, new_ref, z_copy_store_bad_vreg);
+  // Add good colors
+  __ orr(new_ref, Assembler::T16B, new_ref, z_copy_store_good_vreg);
+}
+
+class ZAdjustAddress {
+private:
+  MacroAssembler* _masm;
+  Address _addr;
+  int _pre_adjustment;
+  int _post_adjustment;
+
+  void pre() {
+    if (_pre_adjustment != 0) {
+      _masm->add(_addr.base(), _addr.base(), _addr.offset());
+    }
+  }
+
+  void post() {
+    if (_post_adjustment != 0) {
+      _masm->add(_addr.base(), _addr.base(), _addr.offset());
+    }
+  }
+
+public:
+  ZAdjustAddress(MacroAssembler* masm, Address addr) :
+      _masm(masm),
+      _addr(addr),
+      _pre_adjustment(addr.getMode() == Address::pre ? addr.offset() : 0),
+      _post_adjustment(addr.getMode() == Address::post ? addr.offset() : 0) {
+    pre();
+  }
+
+  ~ZAdjustAddress() {
+    post();
+  }
+
+  Address address() {
+    if (_pre_adjustment != 0 || _post_adjustment != 0) {
+      return Address(_addr.base(), 0);
+    } else {
+      return Address(_addr.base(), _addr.offset());
+    }
+  }
+};
+
+void ZBarrierSetAssembler::copy_load_at(MacroAssembler* masm,
+                                        DecoratorSet decorators,
+                                        BasicType type,
+                                        size_t bytes,
+                                        Register dst1,
+                                        Register dst2,
+                                        Address src,
+                                        Register tmp) {
+  if (!is_reference_type(type)) {
+    BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst1, dst2, src, noreg);
+    return;
+  }
+
+  ZAdjustAddress adjust(masm, src);
+  src = adjust.address();
+
+  BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst1, dst2, src, noreg);
+
+  if (bytes == 8) {
+    copy_load_barrier(masm, dst1, src, tmp);
+  } else if (bytes == 16) {
+    copy_load_barrier(masm, dst1, Address(src.base(), src.offset() + 0), tmp);
+    copy_load_barrier(masm, dst2, Address(src.base(), src.offset() + 8), tmp);
+  } else {
+    ShouldNotReachHere();
+  }
+  if ((decorators & ARRAYCOPY_CHECKCAST) != 0) {
+    __ lsr(dst1, dst1, ZPointerLoadShift);
+  }
+}
+
+void ZBarrierSetAssembler::copy_store_at(MacroAssembler* masm,
+                                         DecoratorSet decorators,
+                                         BasicType type,
+                                         size_t bytes,
+                                         Address dst,
+                                         Register src1,
+                                         Register src2,
+                                         Register tmp1,
+                                         Register tmp2,
+                                         Register tmp3) {
+  if (!is_reference_type(type)) {
+    BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src1, src2, noreg, noreg, noreg);
+    return;
+  }
+
+  ZAdjustAddress adjust(masm, dst);
+  dst = adjust.address();
+
+  if ((decorators & ARRAYCOPY_CHECKCAST) != 0) {
+    __ lsl(src1, src1, ZPointerLoadShift);
+  }
+
+  bool is_dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
+
+  if (is_dest_uninitialized) {
+    __ ldr(tmp1, Address(rthread, ZThreadLocalData::store_good_mask_offset()));
+    if (bytes == 8) {
+      __ bfi(src1, tmp1, 0, 16);
+    } else if (bytes == 16) {
+      __ bfi(src1, tmp1, 0, 16);
+      __ bfi(src2, tmp1, 0, 16);
+    } else {
+      ShouldNotReachHere();
+    }
+  } else {
+    // Store barrier pre values and color new values
+    if (bytes == 8) {
+      __ ldr(tmp1, dst);
+      copy_store_barrier(masm, tmp1, src1, dst, tmp2, tmp3);
+    } else if (bytes == 16) {
+      Address dst1(dst.base(), dst.offset() + 0);
+      Address dst2(dst.base(), dst.offset() + 8);
+
+      __ ldr(tmp1, dst1);
+      copy_store_barrier(masm, tmp1, src1, dst1, tmp2, tmp3);
+
+      __ ldr(tmp1, dst2);
+      copy_store_barrier(masm, tmp1, src2, dst2, tmp2, tmp3);
+    } else {
+      ShouldNotReachHere();
+    }
+  }
+
+  // Store new values
+  BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src1, src2, noreg, noreg, noreg);
+}
+
+void ZBarrierSetAssembler::copy_load_at(MacroAssembler* masm,
+                                        DecoratorSet decorators,
+                                        BasicType type,
+                                        size_t bytes,
+                                        FloatRegister dst1,
+                                        FloatRegister dst2,
+                                        Address src,
+                                        Register tmp1,
+                                        Register tmp2,
+                                        FloatRegister vec_tmp) {
+  if (!is_reference_type(type)) {
+    BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst1, dst2, src, noreg, noreg, fnoreg);
+    return;
+  }
+
+  ZAdjustAddress adjust(masm, src);
+  src = adjust.address();
+
+  BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst1, dst2, src, noreg, noreg, fnoreg);
+
+  if (bytes == 32) {
+    copy_load_barrier(masm, dst1, Address(src.base(), src.offset() + 0), tmp1, tmp2, vec_tmp);
+    copy_load_barrier(masm, dst2, Address(src.base(), src.offset() + 16), tmp1, tmp2, vec_tmp);
+  } else {
+    ShouldNotReachHere();
+  }
+}
+
+void ZBarrierSetAssembler::copy_store_at(MacroAssembler* masm,
+                                         DecoratorSet decorators,
+                                         BasicType type,
+                                         size_t bytes,
+                                         Address dst,
+                                         FloatRegister src1,
+                                         FloatRegister src2,
+                                         Register tmp1,
+                                         Register tmp2,
+                                         Register tmp3,
+                                         FloatRegister vec_tmp1,
+                                         FloatRegister vec_tmp2,
+                                         FloatRegister vec_tmp3) {
+  if (!is_reference_type(type)) {
+    BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src1, src2, noreg, noreg, noreg, fnoreg, fnoreg, fnoreg);
+    return;
+  }
+
+  bool is_dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
+
+  ZAdjustAddress adjust(masm, dst);
+  dst = adjust.address();
+
+  if (is_dest_uninitialized) {
+    if (bytes == 32) {
+      __ bic(src1, Assembler::T16B, src1, z_copy_store_bad_vreg);
+      __ orr(src1, Assembler::T16B, src1, z_copy_store_good_vreg);
+      __ bic(src2, Assembler::T16B, src2, z_copy_store_bad_vreg);
+      __ orr(src2, Assembler::T16B, src2, z_copy_store_good_vreg);
+    } else {
+      ShouldNotReachHere();
+    }
+  } else {
+    // Load pre values
+    BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, vec_tmp1, vec_tmp2, dst, noreg, noreg, fnoreg);
+
+    // Store barrier pre values and color new values
+    if (bytes == 32) {
+      copy_store_barrier(masm, vec_tmp1, src1, Address(dst.base(), dst.offset() + 0), tmp1, tmp2, tmp3, vec_tmp3);
+      copy_store_barrier(masm, vec_tmp2, src2, Address(dst.base(), dst.offset() + 16), tmp1, tmp2, tmp3, vec_tmp3);
+    } else {
+      ShouldNotReachHere();
+    }
+  }
+
+  // Store new values
+  BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src1, src2, noreg, noreg, noreg, fnoreg, fnoreg, fnoreg);
+}
+
 void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
                                                          Register jni_env,
                                                          Register robj,
@@ -184,37 +794,141 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
                                                          Label& slowpath) {
   BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {");
 
-  assert_different_registers(jni_env, robj, tmp);
+  Label done, tagged, weak_tagged, uncolor;
 
-  // Resolve jobject
-  BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath);
+  // Test for tag
+  __ tst(robj, JNIHandles::tag_mask);
+  __ br(Assembler::NE, tagged);
 
-  // The Address offset is too large to direct load - -784. Our range is +127, -128.
-  __ mov(tmp, (int64_t)(in_bytes(ZThreadLocalData::address_bad_mask_offset()) -
-              in_bytes(JavaThread::jni_environment_offset())));
+  // Resolve local handle
+  __ ldr(robj, robj);
+  __ b(done);
 
-  // Load address bad mask
-  __ add(tmp, jni_env, tmp);
-  __ ldr(tmp, Address(tmp));
+  __ bind(tagged);
 
-  // Check address bad mask
+  // Test for weak tag
+  __ tst(robj, JNIHandles::TypeTag::weak_global);
+  __ br(Assembler::NE, weak_tagged);
+
+  // Resolve global handle
+  __ ldr(robj, Address(robj, -JNIHandles::TypeTag::global));
+  __ lea(tmp, load_bad_mask_from_jni_env(jni_env));
+  __ ldr(tmp, tmp);
+  __ tst(robj, tmp);
+  __ br(Assembler::NE, slowpath);
+  __ b(uncolor);
+
+  __ bind(weak_tagged);
+
+  // Resolve weak handle
+  __ ldr(robj, Address(robj, -JNIHandles::TypeTag::weak_global));
+  __ lea(tmp, mark_bad_mask_from_jni_env(jni_env));
+  __ ldr(tmp, tmp);
   __ tst(robj, tmp);
   __ br(Assembler::NE, slowpath);
 
+  __ bind(uncolor);
+
+  // Uncolor
+  __ lsr(robj, robj, ZPointerLoadShift);
+
+  __ bind(done);
+
   BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native");
 }
 
+static uint16_t patch_barrier_relocation_value(int format) {
+  switch (format) {
+  case ZBarrierRelocationFormatLoadGoodBeforeTbX:
+    return (uint16_t)exact_log2(ZPointerRemapped);
+
+  case ZBarrierRelocationFormatMarkBadBeforeMov:
+    return (uint16_t)ZPointerMarkBadMask;
+
+  case ZBarrierRelocationFormatStoreGoodBeforeMov:
+    return (uint16_t)ZPointerStoreGoodMask;
+
+  case ZBarrierRelocationFormatStoreBadBeforeMov:
+    return (uint16_t)ZPointerStoreBadMask;
+
+  default:
+    ShouldNotReachHere();
+    return 0;
+  }
+}
+
+static void change_immediate(uint32_t& instr, uint32_t imm, uint32_t start, uint32_t end) {
+  uint32_t imm_mask = ((1u << start) - 1u) ^ ((1u << (end + 1)) - 1u);
+  instr &= ~imm_mask;
+  instr |= imm << start;
+}
+
+void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
+  const uint16_t value = patch_barrier_relocation_value(format);
+  uint32_t* const patch_addr = (uint32_t*)addr;
+
+  switch (format) {
+  case ZBarrierRelocationFormatLoadGoodBeforeTbX:
+    change_immediate(*patch_addr, value, 19, 23);
+    break;
+  case ZBarrierRelocationFormatStoreGoodBeforeMov:
+  case ZBarrierRelocationFormatMarkBadBeforeMov:
+  case ZBarrierRelocationFormatStoreBadBeforeMov:
+    change_immediate(*patch_addr, value, 5, 20);
+    break;
+  default:
+    ShouldNotReachHere();
+  }
+
+  OrderAccess::fence();
+  ICache::invalidate_word((address)patch_addr);
+}
+
 #ifdef COMPILER1
 
 #undef __
 #define __ ce->masm()->
 
-void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
-                                                         LIR_Opr ref) const {
-  assert_different_registers(rscratch1, rthread, ref->as_register());
+static void z_uncolor(LIR_Assembler* ce, LIR_Opr ref) {
+  __ lsr(ref->as_register(), ref->as_register(), ZPointerLoadShift);
+}
 
-  __ ldr(rscratch1, address_bad_mask_from_thread(rthread));
-  __ tst(ref->as_register(), rscratch1);
+static void z_color(LIR_Assembler* ce, LIR_Opr ref) {
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov);
+  __ movzw(rscratch2, barrier_Relocation::unpatched);
+  __ orr(ref->as_register(), rscratch2, ref->as_register(), Assembler::LSL, ZPointerLoadShift);
+}
+
+void ZBarrierSetAssembler::generate_c1_uncolor(LIR_Assembler* ce, LIR_Opr ref) const {
+  z_uncolor(ce, ref);
+}
+
+void ZBarrierSetAssembler::generate_c1_color(LIR_Assembler* ce, LIR_Opr ref) const {
+  z_color(ce, ref);
+}
+
+void ZBarrierSetAssembler::generate_c1_load_barrier(LIR_Assembler* ce,
+                                                    LIR_Opr ref,
+                                                    ZLoadBarrierStubC1* stub,
+                                                    bool on_non_strong) const {
+
+  if (on_non_strong) {
+    // Test against MarkBad mask
+    assert_different_registers(rscratch1, rthread, ref->as_register());
+    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatMarkBadBeforeMov);
+    __ movzw(rscratch1, barrier_Relocation::unpatched);
+    __ tst(ref->as_register(), rscratch1);
+    __ br(Assembler::NE, *stub->entry());
+    z_uncolor(ce, ref);
+  } else {
+    Label good;
+    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeTbX);
+    __ tbz(ref->as_register(), barrier_Relocation::unpatched, good);
+    __ b(*stub->entry());
+    __ bind(good);
+    z_uncolor(ce, ref);
+  }
+  __ bind(*stub->continuation());
 }
 
 void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
@@ -272,6 +986,57 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
   __ b(*stub->continuation());
 }
 
+void ZBarrierSetAssembler::generate_c1_store_barrier(LIR_Assembler* ce,
+                                                     LIR_Address* addr,
+                                                     LIR_Opr new_zaddress,
+                                                     LIR_Opr new_zpointer,
+                                                     ZStoreBarrierStubC1* stub) const {
+  Register rnew_zaddress = new_zaddress->as_register();
+  Register rnew_zpointer = new_zpointer->as_register();
+
+  store_barrier_fast(ce->masm(),
+                     ce->as_Address(addr),
+                     rnew_zaddress,
+                     rnew_zpointer,
+                     rscratch2,
+                     true,
+                     stub->is_atomic(),
+                     *stub->entry(),
+                     *stub->continuation());
+}
+
+void ZBarrierSetAssembler::generate_c1_store_barrier_stub(LIR_Assembler* ce,
+                                                          ZStoreBarrierStubC1* stub) const {
+  // Stub entry
+  __ bind(*stub->entry());
+  Label slow;
+  Label slow_continuation;
+  store_barrier_medium(ce->masm(),
+                       ce->as_Address(stub->ref_addr()->as_address_ptr()),
+                       rscratch2,
+                       stub->new_zpointer()->as_register(),
+                       rscratch1,
+                       false /* is_native */,
+                       stub->is_atomic(),
+                       *stub->continuation(),
+                       slow,
+                       slow_continuation);
+
+  __ bind(slow);
+
+  __ lea(stub->new_zpointer()->as_register(), ce->as_Address(stub->ref_addr()->as_address_ptr()));
+
+  __ sub(sp, sp, 16);
+  // Setup arguments and call runtime stub
+  assert(stub->new_zpointer()->is_valid(), "invariant");
+  ce->store_parameter(stub->new_zpointer()->as_register(), 0);
+  __ far_call(stub->runtime_stub());
+  __ add(sp, sp, 16);
+
+  // Stub exit
+  __ b(slow_continuation);
+}
+
 #undef __
 #define __ sasm->
 
@@ -291,6 +1056,27 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler*
 
   __ epilogue();
 }
+
+void ZBarrierSetAssembler::generate_c1_store_barrier_runtime_stub(StubAssembler* sasm,
+                                                                  bool self_healing) const {
+  __ prologue("zgc_store_barrier stub", false);
+
+  __ push_call_clobbered_registers();
+
+  // Setup arguments
+  __ load_parameter(0, c_rarg0);
+
+  if (self_healing) {
+    __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr(), 1);
+  } else {
+    __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), 1);
+  }
+
+  __ pop_call_clobbered_registers();
+
+  __ epilogue();
+}
+
 #endif // COMPILER1
 
 #ifdef COMPILER2
@@ -319,7 +1105,7 @@ private:
   PRegSet               _p_regs;
 
 public:
-  void initialize(ZLoadBarrierStubC2* stub) {
+  void initialize(ZBarrierStubC2* stub) {
     // Record registers that needs to be saved/restored
     RegMaskIterator rmi(stub->live());
     while (rmi.has_next()) {
@@ -339,10 +1125,14 @@ public:
     }
 
     // Remove C-ABI SOE registers, scratch regs and _ref register that will be updated
-    _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9, stub->ref());
+    if (stub->result() != noreg) {
+      _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9, stub->result());
+    } else {
+      _gp_regs -= RegSet::range(r19, r30) + RegSet::of(r8, r9);
+    }
   }
 
-  ZSaveLiveRegisters(MacroAssembler* masm, ZLoadBarrierStubC2* stub) :
+  ZSaveLiveRegisters(MacroAssembler* masm, ZBarrierStubC2* stub) :
       _masm(masm),
       _gp_regs(),
       _fp_regs(),
@@ -429,10 +1219,13 @@ public:
 #define __ masm->
 
 void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const {
+  Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
   BLOCK_COMMENT("ZLoadBarrierStubC2");
 
   // Stub entry
-  __ bind(*stub->entry());
+  if (!Compile::current()->output()->in_scratch_emit_size()) {
+    __ bind(*stub->entry());
+  }
 
   {
     ZSaveLiveRegisters save_live_registers(masm, stub);
@@ -444,19 +1237,221 @@ void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, Z
   __ b(*stub->continuation());
 }
 
+void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm, ZStoreBarrierStubC2* stub) const {
+  Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
+  BLOCK_COMMENT("ZStoreBarrierStubC2");
+
+  // Stub entry
+  __ bind(*stub->entry());
+
+  Label slow;
+  Label slow_continuation;
+  store_barrier_medium(masm,
+                       stub->ref_addr(),
+                       stub->new_zpointer(),
+                       rscratch1,
+                       rscratch2,
+                       stub->is_native(),
+                       stub->is_atomic(),
+                       *stub->continuation(),
+                       slow,
+                       slow_continuation);
+
+  __ bind(slow);
+
+  {
+    ZSaveLiveRegisters save_live_registers(masm, stub);
+    __ lea(c_rarg0, stub->ref_addr());
+
+    if (stub->is_native()) {
+      __ lea(rscratch1, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr()));
+    } else if (stub->is_atomic()) {
+      __ lea(rscratch1, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr()));
+    } else {
+      __ lea(rscratch1, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr()));
+    }
+    __ blr(rscratch1);
+  }
+
+  // Stub exit
+  __ b(slow_continuation);
+}
+
+// Only handles forward branch jumps, target_offset >= branch_offset
+static bool aarch64_test_and_branch_reachable(int branch_offset, int target_offset) {
+  assert(branch_offset >= 0, "branch to stub offsets must be positive");
+  assert(target_offset >= 0, "offset in stubs section must be positive");
+  assert(target_offset >= branch_offset, "forward branches only, branch_offset -> target_offset");
+
+  const int test_and_branch_delta_limit = 32 * K;
+
+  const int test_and_branch_to_trampoline_delta = target_offset - branch_offset;
+
+  return test_and_branch_to_trampoline_delta < test_and_branch_delta_limit;
+}
+
+ZLoadBarrierStubC2Aarch64::ZLoadBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register ref, int offset)
+  : ZLoadBarrierStubC2(node, ref_addr, ref), _test_and_branch_reachable_entry(), _offset(offset), _deferred_emit(false), _test_and_branch_reachable(false) {
+  PhaseOutput* const output = Compile::current()->output();
+  if (output->in_scratch_emit_size()) {
+    return;
+  }
+  const int code_size = output->buffer_sizing_data()->_code;
+  const int offset_code = _offset;
+  // Assumption that the stub can always be reached from a branch immediate. (128 M Product, 2 M Debug)
+  // Same assumption is made in z_aarch64.ad
+  const int trampoline_offset = trampoline_stubs_count() * NativeInstruction::instruction_size;
+  _test_and_branch_reachable = aarch64_test_and_branch_reachable(offset_code, code_size + trampoline_offset);
+  if (_test_and_branch_reachable) {
+    inc_trampoline_stubs_count();
+  }
+}
+
+int ZLoadBarrierStubC2Aarch64::get_stub_size() {
+  PhaseOutput* const output = Compile::current()->output();
+  assert(!output->in_scratch_emit_size(), "only used when emitting stubs");
+  BufferBlob* const blob = output->scratch_buffer_blob();
+  CodeBuffer cb(blob->content_begin(), (address)output->scratch_locs_memory() - blob->content_begin());
+  MacroAssembler masm(&cb);
+  output->set_in_scratch_emit_size(true);
+  ZLoadBarrierStubC2::emit_code(masm);
+  output->set_in_scratch_emit_size(false);
+  return cb.insts_size();
+}
+
+ZLoadBarrierStubC2Aarch64* ZLoadBarrierStubC2Aarch64::create(const MachNode* node, Address ref_addr, Register ref, int offset) {
+  ZLoadBarrierStubC2Aarch64* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2Aarch64(node, ref_addr, ref, offset);
+  register_stub(stub);
+  return stub;
+}
+
+#undef __
+#define __ masm.
+
+void ZLoadBarrierStubC2Aarch64::emit_code(MacroAssembler& masm) {
+  PhaseOutput* const output = Compile::current()->output();
+  const int branch_offset = _offset;
+  const int target_offset = __ offset();
+
+  // Deferred emission, emit actual stub
+  if (_deferred_emit) {
+    ZLoadBarrierStubC2::emit_code(masm);
+    return;
+  }
+  _deferred_emit = true;
+
+  // No trampoline used, defer emission to after trampolines
+  if (!_test_and_branch_reachable) {
+    register_stub(this);
+    return;
+  }
+
+  // Current assumption is that the barrier stubs are the first stubs emitted after the actual code
+  assert(stubs_start_offset() <= output->buffer_sizing_data()->_code, "stubs are assumed to be emitted directly after code and code_size is a hard limit on where it can start");
+
+  __ bind(_test_and_branch_reachable_entry);
+
+  // Next branch's offset is unknown, but is > branch_offset
+  const int next_branch_offset = branch_offset + NativeInstruction::instruction_size;
+  // If emitting the stub directly does not interfere with emission of the next trampoline then do it to avoid a double jump.
+  if (aarch64_test_and_branch_reachable(next_branch_offset, target_offset + get_stub_size())) {
+    // The next potential trampoline will still be reachable even if we emit the whole stub
+    ZLoadBarrierStubC2::emit_code(masm);
+  } else {
+    // Emit trampoline and defer actual stub to the end
+    assert(aarch64_test_and_branch_reachable(branch_offset, target_offset), "trampoline should be reachable");
+    __ b(*ZLoadBarrierStubC2::entry());
+    register_stub(this);
+  }
+}
+
+bool ZLoadBarrierStubC2Aarch64::is_test_and_branch_reachable() {
+  return _test_and_branch_reachable;
+}
+
+Label* ZLoadBarrierStubC2Aarch64::entry() {
+  if (_test_and_branch_reachable) {
+    return &_test_and_branch_reachable_entry;
+  }
+  return ZBarrierStubC2::entry();
+}
+
+ZStoreBarrierStubC2Aarch64::ZStoreBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic)
+  : ZStoreBarrierStubC2(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic), _deferred_emit(false) {}
+
+ZStoreBarrierStubC2Aarch64* ZStoreBarrierStubC2Aarch64::create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic) {
+  ZStoreBarrierStubC2Aarch64* const stub = new (Compile::current()->comp_arena()) ZStoreBarrierStubC2Aarch64(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic);
+  register_stub(stub);
+  return stub;
+}
+
+void ZStoreBarrierStubC2Aarch64::emit_code(MacroAssembler& masm) {
+  if (_deferred_emit) {
+    ZStoreBarrierStubC2::emit_code(masm);
+    return;
+  }
+  // Defer emission of store barriers so that trampolines are emitted first
+  _deferred_emit = true;
+  register_stub(this);
+}
+
+#undef __
+
 #endif // COMPILER2
 
 #undef __
 #define __ masm->
 
 void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
-  // Check if mask is good.
-  // verifies that ZAddressBadMask & r0 == 0
-  __ ldr(tmp2, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
-  __ andr(tmp1, obj, tmp2);
-  __ cbnz(tmp1, error);
+  // C1 calls verfy_oop in the middle of barriers, before they have been uncolored
+  // and after being colored. Therefore, we must deal with colored oops as well.
+  Label done;
+  Label check_oop;
+  Label check_zaddress;
+  int color_bits = ZPointerRemappedShift + ZPointerRemappedBits;
 
-  BarrierSetAssembler::check_oop(masm, obj, tmp1, tmp2, error);
+  uintptr_t shifted_base_start_mask = (UCONST64(1) << (ZAddressHeapBaseShift + color_bits + 1)) - 1;
+  uintptr_t shifted_base_end_mask = (UCONST64(1) << (ZAddressHeapBaseShift + 1)) - 1;
+  uintptr_t shifted_base_mask = shifted_base_start_mask ^ shifted_base_end_mask;
+
+  uintptr_t shifted_address_end_mask = (UCONST64(1) << (color_bits + 1)) - 1;
+  uintptr_t shifted_address_mask = shifted_address_end_mask ^ (uintptr_t)CONST64(-1);
+
+  __ get_nzcv(tmp2);
+
+  // Check colored null
+  __ mov(tmp1, shifted_address_mask);
+  __ tst(tmp1, obj);
+  __ br(Assembler::EQ, done);
+
+  // Check for zpointer
+  __ mov(tmp1, shifted_base_mask);
+  __ tst(tmp1, obj);
+  __ br(Assembler::EQ, check_oop);
+
+  // Uncolor presumed zpointer
+  __ lsr(obj, obj, ZPointerLoadShift);
+
+  __ b(check_zaddress);
+
+  __ bind(check_oop);
+
+  // make sure klass is 'reasonable', which is not zero.
+  __ load_klass(tmp1, obj);  // get klass
+  __ tst(tmp1, tmp1);
+  __ br(Assembler::EQ, error); // if klass is null it is broken
+
+  __ bind(check_zaddress);
+  // Check if the oop is in the right area of memory
+  __ mov(tmp1, (intptr_t) Universe::verify_oop_mask());
+  __ andr(tmp1, tmp1, obj);
+  __ mov(obj, (intptr_t) Universe::verify_oop_bits());
+  __ cmp(tmp1, obj);
+  __ br(Assembler::NE, error);
+
+  __ bind(done);
+
+  __ set_nzcv(tmp2);
 }
 
 #undef __
diff --git a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
index c852dac3a4d..00714e5c0c0 100644
--- a/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/z/zBarrierSetAssembler_aarch64.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,21 +27,32 @@
 #include "code/vmreg.hpp"
 #include "oops/accessDecorators.hpp"
 #ifdef COMPILER2
+#include "gc/z/c2/zBarrierSetC2.hpp"
 #include "opto/optoreg.hpp"
 #endif // COMPILER2
 
 #ifdef COMPILER1
+class LIR_Address;
 class LIR_Assembler;
 class LIR_Opr;
 class StubAssembler;
 class ZLoadBarrierStubC1;
+class ZStoreBarrierStubC1;
 #endif // COMPILER1
 
 #ifdef COMPILER2
+class MachNode;
 class Node;
-class ZLoadBarrierStubC2;
 #endif // COMPILER2
 
+// ZBarrierRelocationFormatLoadGoodBeforeTbX is used for both tbnz and tbz
+// They are patched in the same way, their immediate value has the same
+// structure
+const int ZBarrierRelocationFormatLoadGoodBeforeTbX  = 0;
+const int ZBarrierRelocationFormatMarkBadBeforeMov   = 1;
+const int ZBarrierRelocationFormatStoreGoodBeforeMov = 2;
+const int ZBarrierRelocationFormatStoreBadBeforeMov  = 3;
+
 class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase {
 public:
   virtual void load_at(MacroAssembler* masm,
@@ -52,7 +63,27 @@ public:
                        Register tmp1,
                        Register tmp2);
 
-#ifdef ASSERT
+  void store_barrier_fast(MacroAssembler* masm,
+                          Address ref_addr,
+                          Register rnew_zaddress,
+                          Register rnew_zpointer,
+                          Register rtmp,
+                          bool in_nmethod,
+                          bool is_atomic,
+                          Label& medium_path,
+                          Label& medium_path_continuation) const;
+
+  void store_barrier_medium(MacroAssembler* masm,
+                            Address ref_addr,
+                            Register rtmp1,
+                            Register rtmp2,
+                            Register rtmp3,
+                            bool is_native,
+                            bool is_atomic,
+                            Label& medium_path_continuation,
+                            Label& slow_path,
+                            Label& slow_path_continuation) const;
+
   virtual void store_at(MacroAssembler* masm,
                         DecoratorSet decorators,
                         BasicType type,
@@ -61,7 +92,6 @@ public:
                         Register tmp1,
                         Register tmp2,
                         Register tmp3);
-#endif // ASSERT
 
   virtual void arraycopy_prologue(MacroAssembler* masm,
                                   DecoratorSet decorators,
@@ -71,23 +101,89 @@ public:
                                   Register count,
                                   RegSet saved_regs);
 
+  virtual void copy_load_at(MacroAssembler* masm,
+                            DecoratorSet decorators,
+                            BasicType type,
+                            size_t bytes,
+                            Register dst1,
+                            Register dst2,
+                            Address src,
+                            Register tmp);
+
+  virtual void copy_store_at(MacroAssembler* masm,
+                             DecoratorSet decorators,
+                             BasicType type,
+                             size_t bytes,
+                             Address dst,
+                             Register src1,
+                             Register src2,
+                             Register tmp1,
+                             Register tmp2,
+                             Register tmp3);
+
+  virtual void copy_load_at(MacroAssembler* masm,
+                            DecoratorSet decorators,
+                            BasicType type,
+                            size_t bytes,
+                            FloatRegister dst1,
+                            FloatRegister dst2,
+                            Address src,
+                            Register tmp1,
+                            Register tmp2,
+                            FloatRegister vec_tmp);
+
+  virtual void copy_store_at(MacroAssembler* masm,
+                             DecoratorSet decorators,
+                             BasicType type,
+                             size_t bytes,
+                             Address dst,
+                             FloatRegister src1,
+                             FloatRegister src2,
+                             Register tmp1,
+                             Register tmp2,
+                             Register tmp3,
+                             FloatRegister vec_tmp1,
+                             FloatRegister vec_tmp2,
+                             FloatRegister vec_tmp3);
+
   virtual void try_resolve_jobject_in_native(MacroAssembler* masm,
                                              Register jni_env,
                                              Register robj,
                                              Register tmp,
                                              Label& slowpath);
 
-  virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
+  virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_instruction_and_data_patch; }
+
+  void patch_barrier_relocation(address addr, int format);
+
+  void patch_barriers() {}
 
 #ifdef COMPILER1
-  void generate_c1_load_barrier_test(LIR_Assembler* ce,
-                                     LIR_Opr ref) const;
+  void generate_c1_color(LIR_Assembler* ce, LIR_Opr ref) const;
+  void generate_c1_uncolor(LIR_Assembler* ce, LIR_Opr ref) const;
+
+  void generate_c1_load_barrier(LIR_Assembler* ce,
+                                LIR_Opr ref,
+                                ZLoadBarrierStubC1* stub,
+                                bool on_non_strong) const;
 
   void generate_c1_load_barrier_stub(LIR_Assembler* ce,
                                      ZLoadBarrierStubC1* stub) const;
 
   void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
                                              DecoratorSet decorators) const;
+
+  void generate_c1_store_barrier(LIR_Assembler* ce,
+                                 LIR_Address* addr,
+                                 LIR_Opr new_zaddress,
+                                 LIR_Opr new_zpointer,
+                                 ZStoreBarrierStubC1* stub) const;
+
+  void generate_c1_store_barrier_stub(LIR_Assembler* ce,
+                                      ZStoreBarrierStubC1* stub) const;
+
+  void generate_c1_store_barrier_runtime_stub(StubAssembler* sasm,
+                                              bool self_healing) const;
 #endif // COMPILER1
 
 #ifdef COMPILER2
@@ -96,9 +192,103 @@ public:
 
   void generate_c2_load_barrier_stub(MacroAssembler* masm,
                                      ZLoadBarrierStubC2* stub) const;
+  void generate_c2_store_barrier_stub(MacroAssembler* masm,
+                                      ZStoreBarrierStubC2* stub) const;
 #endif // COMPILER2
 
   void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
 };
 
+#ifdef COMPILER2
+
+// Load barriers on aarch64 are implemented with a test-and-branch immediate instruction.
+// This immediate has a max delta of 32K. Because of this the branch is implemented with
+// a small jump, as follows:
+//      __ tbz(ref, barrier_Relocation::unpatched, good);
+//      __ b(*stub->entry());
+//      __ bind(good);
+//
+// If we can guarantee that the *stub->entry() label is within 32K we can replace the above
+// code with:
+//      __ tbnz(ref, barrier_Relocation::unpatched, *stub->entry());
+//
+// From the branch shortening part of PhaseOutput we get a pessimistic code size that the code
+// will not grow beyond.
+//
+// The stubs objects are created and registered when the load barriers are emitted. The decision
+// between emitting the long branch or the test and branch is done at this point and uses the
+// pessimistic code size from branch shortening.
+//
+// After the code has been emitted the barrier set will emit all the stubs. When the stubs are
+// emitted we know the real code size. Because of this the trampoline jump can be skipped in
+// favour of emitting the stub directly if it does not interfere with the next trampoline stub.
+// (With respect to test and branch distance)
+//
+// The algorithm for emitting the load barrier branches and stubs now have three versions
+// depending on the distance between the barrier and the stub.
+// Version 1: Not Reachable with a test-and-branch immediate
+// Version 2: Reachable with a test-and-branch immediate via trampoline
+// Version 3: Reachable with a test-and-branch immediate without trampoline
+//
+//     +--------------------- Code ----------------------+
+//     |                      ***                        |
+//     | b(stub1)                                        | (Version 1)
+//     |                      ***                        |
+//     | tbnz(ref, barrier_Relocation::unpatched, tramp) | (Version 2)
+//     |                      ***                        |
+//     | tbnz(ref, barrier_Relocation::unpatched, stub3) | (Version 3)
+//     |                      ***                        |
+//     +--------------------- Stub ----------------------+
+//     | tramp: b(stub2)                                 | (Trampoline slot)
+//     | stub3:                                          |
+//     |                  * Stub Code*                   |
+//     | stub1:                                          |
+//     |                  * Stub Code*                   |
+//     | stub2:                                          |
+//     |                  * Stub Code*                   |
+//     +-------------------------------------------------+
+//
+//  Version 1: Is emitted if the pessimistic distance between the branch instruction and the current
+//             trampoline slot cannot fit in a test and branch immediate.
+//
+//  Version 2: Is emitted if the distance between the branch instruction and the current trampoline
+//             slot can fit in a test and branch immediate. But emitting the stub directly would
+//             interfere with the next trampoline.
+//
+//  Version 3: Same as version two but emitting the stub directly (skipping the trampoline) does not
+//             interfere with the next trampoline.
+//
+class ZLoadBarrierStubC2Aarch64 : public ZLoadBarrierStubC2 {
+private:
+  Label _test_and_branch_reachable_entry;
+  const int _offset;
+  bool _deferred_emit;
+  bool _test_and_branch_reachable;
+
+  ZLoadBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register ref, int offset);
+
+  int get_stub_size();
+public:
+  static ZLoadBarrierStubC2Aarch64* create(const MachNode* node, Address ref_addr, Register ref, int offset);
+
+  virtual void emit_code(MacroAssembler& masm);
+  bool is_test_and_branch_reachable();
+  Label* entry();
+};
+
+
+class ZStoreBarrierStubC2Aarch64 : public ZStoreBarrierStubC2 {
+private:
+  bool _deferred_emit;
+
+  ZStoreBarrierStubC2Aarch64(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic);
+
+public:
+  static ZStoreBarrierStubC2Aarch64* create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic);
+
+  virtual void emit_code(MacroAssembler& masm);
+};
+
+#endif // COMPILER2
+
 #endif // CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.hpp
index 6ced6afe02a..b5f2f8525c6 100644
--- a/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/gc/z/zGlobals_aarch64.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,10 +24,8 @@
 #ifndef CPU_AARCH64_GC_Z_ZGLOBALS_AARCH64_HPP
 #define CPU_AARCH64_GC_Z_ZGLOBALS_AARCH64_HPP
 
-const size_t ZPlatformHeapViews        = 3;
+#include "utilities/globalDefinitions.hpp"
+
 const size_t ZPlatformCacheLineSize    = 64;
 
-size_t ZPlatformAddressOffsetBits();
-size_t ZPlatformAddressMetadataShift();
-
 #endif // CPU_AARCH64_GC_Z_ZGLOBALS_AARCH64_HPP
diff --git a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad
index bd1c2cc9f93..8c698635ad0 100644
--- a/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad
+++ b/src/hotspot/cpu/aarch64/gc/z/z_aarch64.ad
@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
@@ -31,30 +31,79 @@ source_hpp %{
 
 source %{
 
-static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data) {
-  if (barrier_data == ZLoadBarrierElided) {
-    return;
-  }
-  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
-  __ ldr(tmp, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
-  __ andr(tmp, tmp, ref);
-  __ cbnz(tmp, *stub->entry());
+#include "gc/z/zBarrierSetAssembler.hpp"
+
+static void z_color(MacroAssembler& _masm, const MachNode* node, Register dst, Register src) {
+  assert_different_registers(src, dst);
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBeforeMov);
+  __ movzw(dst, barrier_Relocation::unpatched);
+  __ orr(dst, dst, src, Assembler::LSL, ZPointerLoadShift);
+}
+
+static void z_uncolor(MacroAssembler& _masm, const MachNode* node, Register ref) {
+  __ lsr(ref, ref, ZPointerLoadShift);
+}
+
+static void z_keep_alive_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatMarkBadBeforeMov);
+  __ movzw(tmp, barrier_Relocation::unpatched);
+  __ tst(ref, tmp);
+  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref);
+  __ br(Assembler::NE, *stub->entry());
+  z_uncolor(_masm, node, ref);
   __ bind(*stub->continuation());
 }
 
-static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
-  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, ZLoadBarrierStrong);
-  __ b(*stub->entry());
+static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
+  Assembler::InlineSkippedInstructionsCounter skipped_counter(&_masm);
+  const bool on_non_strong =
+      ((node->barrier_data() & ZBarrierWeak) != 0) ||
+      ((node->barrier_data() & ZBarrierPhantom) != 0);
+
+  if (on_non_strong) {
+    z_keep_alive_load_barrier(_masm, node, ref_addr, ref, tmp);
+    return;
+  }
+
+  if (node->barrier_data() == ZBarrierElided) {
+    z_uncolor(_masm, node, ref);
+    return;
+  }
+
+  ZLoadBarrierStubC2Aarch64* const stub = ZLoadBarrierStubC2Aarch64::create(node, ref_addr, ref, __ offset());
+  if (stub->is_test_and_branch_reachable()) {
+    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeTbX);
+    __ tbnz(ref, barrier_Relocation::unpatched, *stub->entry());
+  } else {
+    Label good;
+    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeTbX);
+    __ tbz(ref, barrier_Relocation::unpatched, good);
+    __ b(*stub->entry());
+    __ bind(good);
+  }
+  z_uncolor(_masm, node, ref);
   __ bind(*stub->continuation());
 }
 
+static void z_store_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register rnew_zaddress, Register rnew_zpointer, Register tmp, bool is_atomic) {
+  Assembler::InlineSkippedInstructionsCounter skipped_counter(&_masm);
+  if (node->barrier_data() == ZBarrierElided) {
+    z_color(_masm, node, rnew_zpointer, rnew_zaddress);
+  } else {
+    bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
+    ZStoreBarrierStubC2Aarch64* const stub = ZStoreBarrierStubC2Aarch64::create(node, ref_addr, rnew_zaddress, rnew_zpointer, is_native, is_atomic);
+    ZBarrierSetAssembler* bs_asm = ZBarrierSet::assembler();
+    bs_asm->store_barrier_fast(&_masm, ref_addr, rnew_zaddress, rnew_zpointer, tmp, true /* in_nmethod */, is_atomic, *stub->entry(), *stub->continuation());
+  }
+}
+
 %}
 
 // Load Pointer
 instruct zLoadP(iRegPNoSp dst, memory8 mem, rFlagsReg cr)
 %{
   match(Set dst (LoadP mem));
-  predicate(UseZGC && !needs_acquiring_load(n) && (n->as_Load()->barrier_data() != 0));
+  predicate(UseZGC && ZGenerational && !needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
   effect(TEMP dst, KILL cr);
 
   ins_cost(4 * INSN_COST);
@@ -64,7 +113,7 @@ instruct zLoadP(iRegPNoSp dst, memory8 mem, rFlagsReg cr)
   ins_encode %{
     const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
     __ ldr($dst$$Register, ref_addr);
-    z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch2 /* tmp */, barrier_data());
+    z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch1);
   %}
 
   ins_pipe(iload_reg_mem);
@@ -74,7 +123,7 @@ instruct zLoadP(iRegPNoSp dst, memory8 mem, rFlagsReg cr)
 instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg cr)
 %{
   match(Set dst (LoadP mem));
-  predicate(UseZGC && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
+  predicate(UseZGC && ZGenerational && needs_acquiring_load(n) && n->as_Load()->barrier_data() != 0);
   effect(TEMP dst, KILL cr);
 
   ins_cost(VOLATILE_REF_COST);
@@ -82,18 +131,53 @@ instruct zLoadPVolatile(iRegPNoSp dst, indirect mem /* sync_memory */, rFlagsReg
   format %{ "ldar  $dst, $mem\t" %}
 
   ins_encode %{
+    const Address ref_addr = Address($mem$$Register);
     __ ldar($dst$$Register, $mem$$Register);
-    z_load_barrier(_masm, this, Address($mem$$Register), $dst$$Register, rscratch2 /* tmp */, barrier_data());
+    z_load_barrier(_masm, this, ref_addr, $dst$$Register, rscratch1);
   %}
 
   ins_pipe(pipe_serial);
 %}
 
-instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+// Store Pointer
+instruct zStoreP(memory mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
+%{
+  predicate(UseZGC && ZGenerational && !needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
+  match(Set mem (StoreP mem src));
+  effect(TEMP tmp, KILL cr);
+
+  ins_cost(125); // XXX
+  format %{ "movq    $mem, $src\t# ptr" %}
+  ins_encode %{
+    const Address ref_addr = mem2address($mem->opcode(), as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
+    z_store_barrier(_masm, this, ref_addr, $src$$Register, $tmp$$Register, rscratch2, false /* is_atomic */);
+    __ str($tmp$$Register, ref_addr);
+  %}
+  ins_pipe(pipe_serial);
+%}
+
+// Store Pointer Volatile
+instruct zStorePVolatile(indirect mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
+%{
+  predicate(UseZGC && ZGenerational && needs_releasing_store(n) && n->as_Store()->barrier_data() != 0);
+  match(Set mem (StoreP mem src));
+  effect(TEMP tmp, KILL cr);
+
+  ins_cost(125); // XXX
+  format %{ "movq    $mem, $src\t# ptr" %}
+  ins_encode %{
+    const Address ref_addr = Address($mem$$Register);
+    z_store_barrier(_masm, this, ref_addr, $src$$Register, $tmp$$Register, rscratch2, false /* is_atomic */);
+    __ stlr($tmp$$Register, $mem$$Register);
+  %}
+  ins_pipe(pipe_serial);
+%}
+
+instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
-  predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
-  effect(KILL cr, TEMP_DEF res);
+  predicate(UseZGC && ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
@@ -102,108 +186,83 @@ instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
 
   ins_encode %{
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
-    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
-               false /* acquire */, true /* release */, false /* weak */, rscratch2);
+    Address ref_addr($mem$$Register);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register);
+    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
+               false /* acquire */, true /* release */, false /* weak */, noreg);
     __ cset($res$$Register, Assembler::EQ);
-    if (barrier_data() != ZLoadBarrierElided) {
-      Label good;
-      __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
-      __ andr(rscratch1, rscratch1, rscratch2);
-      __ cbz(rscratch1, good);
-      z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */);
-      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
-                 false /* acquire */, true /* release */, false /* weak */, rscratch2);
-      __ cset($res$$Register, Assembler::EQ);
-      __ bind(good);
-    }
   %}
 
   ins_pipe(pipe_slow);
 %}
 
-instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
-  predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong));
-  effect(KILL cr, TEMP_DEF res);
+  predicate(UseZGC && ZGenerational && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
- format %{ "cmpxchg $mem, $oldval, $newval\n\t"
-           "cset    $res, EQ" %}
+  format %{ "cmpxchg $mem, $oldval, $newval\n\t"
+            "cset    $res, EQ" %}
 
   ins_encode %{
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
-    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
-               true /* acquire */, true /* release */, false /* weak */, rscratch2);
+    Address ref_addr($mem$$Register);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register);
+    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
+               true /* acquire */, true /* release */, false /* weak */, noreg);
     __ cset($res$$Register, Assembler::EQ);
-    if (barrier_data() != ZLoadBarrierElided) {
-      Label good;
-      __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
-      __ andr(rscratch1, rscratch1, rscratch2);
-      __ cbz(rscratch1, good);
-      z_load_barrier_slow_path(_masm, this, Address($mem$$Register), rscratch2 /* ref */, rscratch1 /* tmp */ );
-      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
-                 true /* acquire */, true /* release */, false /* weak */, rscratch2);
-      __ cset($res$$Register, Assembler::EQ);
-      __ bind(good);
-    }
   %}
 
   ins_pipe(pipe_slow);
 %}
 
-instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+
+instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
-  predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
-  effect(TEMP_DEF res, KILL cr);
+  predicate(UseZGC && ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
-  format %{ "cmpxchg $res = $mem, $oldval, $newval" %}
+  format %{ "cmpxchg $mem, $oldval, $newval\n\t"
+            "cset    $res, EQ" %}
 
   ins_encode %{
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
-    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+    Address ref_addr($mem$$Register);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register);
+    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
                false /* acquire */, true /* release */, false /* weak */, $res$$Register);
-    if (barrier_data() != ZLoadBarrierElided) {
-      Label good;
-      __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
-      __ andr(rscratch1, rscratch1, $res$$Register);
-      __ cbz(rscratch1, good);
-      z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */);
-      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
-                 false /* acquire */, true /* release */, false /* weak */, $res$$Register);
-      __ bind(good);
-    }
+    z_uncolor(_masm, this, $res$$Register);
   %}
 
   ins_pipe(pipe_slow);
 %}
 
-instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
-  predicate(UseZGC && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
-  effect(TEMP_DEF res, KILL cr);
+  predicate(UseZGC && ZGenerational && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, TEMP res, KILL cr);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
-  format %{ "cmpxchg $res = $mem, $oldval, $newval" %}
+  format %{ "cmpxchg $mem, $oldval, $newval\n\t"
+            "cset    $res, EQ" %}
 
   ins_encode %{
     guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
-    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
+    Address ref_addr($mem$$Register);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, rscratch2, true /* is_atomic */);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register);
+    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::xword,
                true /* acquire */, true /* release */, false /* weak */, $res$$Register);
-    if (barrier_data() != ZLoadBarrierElided) {
-      Label good;
-      __ ldr(rscratch1, Address(rthread, ZThreadLocalData::address_bad_mask_offset()));
-      __ andr(rscratch1, rscratch1, $res$$Register);
-      __ cbz(rscratch1, good);
-      z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, rscratch1 /* tmp */);
-      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::xword,
-                 true /* acquire */, true /* release */, false /* weak */, $res$$Register);
-      __ bind(good);
-    }
+    z_uncolor(_masm, this, $res$$Register);
   %}
 
   ins_pipe(pipe_slow);
@@ -211,16 +270,17 @@ instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
 
 instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
   match(Set prev (GetAndSetP mem newv));
-  predicate(UseZGC && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
-  effect(TEMP_DEF prev, KILL cr);
+  predicate(UseZGC && ZGenerational && !needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP prev, KILL cr);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
   format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 
   ins_encode %{
-    __ atomic_xchg($prev$$Register, $newv$$Register, $mem$$Register);
-    z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, barrier_data());
+    z_store_barrier(_masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, rscratch2, true /* is_atomic */);
+    __ atomic_xchg($prev$$Register, $prev$$Register, $mem$$Register);
+    z_uncolor(_masm, this, $prev$$Register);
   %}
 
   ins_pipe(pipe_serial);
@@ -228,16 +288,18 @@ instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
 
 instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
   match(Set prev (GetAndSetP mem newv));
-  predicate(UseZGC && needs_acquiring_load_exclusive(n) && (n->as_LoadStore()->barrier_data() != 0));
-  effect(TEMP_DEF prev, KILL cr);
+  predicate(UseZGC && ZGenerational && needs_acquiring_load_exclusive(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP prev, KILL cr);
 
-  ins_cost(VOLATILE_REF_COST);
+  ins_cost(2 * VOLATILE_REF_COST);
 
-  format %{ "atomic_xchg_acq  $prev, $newv, [$mem]" %}
+  format %{ "atomic_xchg  $prev, $newv, [$mem]" %}
 
   ins_encode %{
-    __ atomic_xchgal($prev$$Register, $newv$$Register, $mem$$Register);
-    z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, rscratch2 /* tmp */, barrier_data());
+    z_store_barrier(_masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, rscratch2, true /* is_atomic */);
+    __ atomic_xchgal($prev$$Register, $prev$$Register, $mem$$Register);
+    z_uncolor(_masm, this, $prev$$Register);
   %}
+
   ins_pipe(pipe_serial);
 %}
diff --git a/src/hotspot/cpu/aarch64/relocInfo_aarch64.hpp b/src/hotspot/cpu/aarch64/relocInfo_aarch64.hpp
index 904fe081427..47cfc86d17f 100644
--- a/src/hotspot/cpu/aarch64/relocInfo_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/relocInfo_aarch64.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -33,7 +33,8 @@
     // the two lowest offset bits can always be discarded.
     offset_unit        =  4,
     // Must be at least 1 for RelocInfo::narrow_oop_in_const.
-    format_width       =  1
+    // Must be at least 2 for ZGC GC barrier patching.
+    format_width       =  2
   };
 
  public:
diff --git a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp
index fc93e14c080..76ea4cad783 100644
--- a/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp
+++ b/src/hotspot/cpu/aarch64/stubRoutines_aarch64.hpp
@@ -38,8 +38,8 @@ enum platform_dependent_constants {
   // simply increase sizes if too small (assembler will crash if too small)
   _initial_stubs_code_size      = 10000,
   _continuation_stubs_code_size =  2000,
-  _compiler_stubs_code_size     = 30000,
-  _final_stubs_code_size        = 20000
+  _compiler_stubs_code_size     = 30000 ZGC_ONLY(+10000),
+  _final_stubs_code_size        = 20000 ZGC_ONLY(+60000)
 };
 
 class aarch64 {
diff --git a/src/hotspot/cpu/ppc/assembler_ppc.hpp b/src/hotspot/cpu/ppc/assembler_ppc.hpp
index 094d894d2da..afb0164d3d3 100644
--- a/src/hotspot/cpu/ppc/assembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/assembler_ppc.hpp
@@ -237,10 +237,12 @@ class Assembler : public AbstractAssembler {
 
   enum opcdxos_masks {
     XL_FORM_OPCODE_MASK = (63u << OPCODE_SHIFT) | (1023u << 1),
+    ANDI_OPCODE_MASK    = (63u << OPCODE_SHIFT),
     ADDI_OPCODE_MASK    = (63u << OPCODE_SHIFT),
     ADDIS_OPCODE_MASK   = (63u << OPCODE_SHIFT),
     BXX_OPCODE_MASK     = (63u << OPCODE_SHIFT),
     BCXX_OPCODE_MASK    = (63u << OPCODE_SHIFT),
+    CMPLI_OPCODE_MASK   = (63u << OPCODE_SHIFT),
     // trap instructions
     TDI_OPCODE_MASK     = (63u << OPCODE_SHIFT),
     TWI_OPCODE_MASK     = (63u << OPCODE_SHIFT),
@@ -1478,6 +1480,9 @@ class Assembler : public AbstractAssembler {
   static bool is_addis(int x) {
      return ADDIS_OPCODE == (x & ADDIS_OPCODE_MASK);
   }
+  static bool is_andi(int x) {
+     return ANDI_OPCODE == (x & ANDI_OPCODE_MASK);
+  }
   static bool is_bxx(int x) {
      return BXX_OPCODE == (x & BXX_OPCODE_MASK);
   }
@@ -1502,6 +1507,9 @@ class Assembler : public AbstractAssembler {
   static bool is_bclr(int x) {
      return BCLR_OPCODE == (x & XL_FORM_OPCODE_MASK);
   }
+  static bool is_cmpli(int x) {
+     return CMPLI_OPCODE == (x & CMPLI_OPCODE_MASK);
+  }
   static bool is_li(int x) {
      return is_addi(x) && inv_ra_field(x)==0;
   }
diff --git a/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.cpp
new file mode 100644
index 00000000000..b83994ee8de
--- /dev/null
+++ b/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.cpp
@@ -0,0 +1,585 @@
+/*
+ * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2022 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "asm/register.hpp"
+#include "precompiled.hpp"
+#include "asm/macroAssembler.inline.hpp"
+#include "code/codeBlob.hpp"
+#include "code/vmreg.inline.hpp"
+#include "gc/x/xBarrier.inline.hpp"
+#include "gc/x/xBarrierSet.hpp"
+#include "gc/x/xBarrierSetAssembler.hpp"
+#include "gc/x/xBarrierSetRuntime.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+#include "memory/resourceArea.hpp"
+#include "register_ppc.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/macros.hpp"
+#ifdef COMPILER1
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/x/c1/xBarrierSetC1.hpp"
+#endif // COMPILER1
+#ifdef COMPILER2
+#include "gc/x/c2/xBarrierSetC2.hpp"
+#endif // COMPILER2
+
+#undef __
+#define __ masm->
+
+void XBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                   Register base, RegisterOrConstant ind_or_offs, Register dst,
+                                   Register tmp1, Register tmp2,
+                                   MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null) {
+  __ block_comment("load_at (zgc) {");
+
+  // Check whether a special gc barrier is required for this particular load
+  // (e.g. whether it's a reference load or not)
+  if (!XBarrierSet::barrier_needed(decorators, type)) {
+    BarrierSetAssembler::load_at(masm, decorators, type, base, ind_or_offs, dst,
+                                 tmp1, tmp2, preservation_level, L_handle_null);
+    return;
+  }
+
+  if (ind_or_offs.is_register()) {
+    assert_different_registers(base, ind_or_offs.as_register(), tmp1, tmp2, R0, noreg);
+    assert_different_registers(dst, ind_or_offs.as_register(), tmp1, tmp2, R0, noreg);
+  } else {
+    assert_different_registers(base, tmp1, tmp2, R0, noreg);
+    assert_different_registers(dst, tmp1, tmp2, R0, noreg);
+  }
+
+  /* ==== Load the pointer using the standard implementation for the actual heap access
+          and the decompression of compressed pointers ==== */
+  // Result of 'load_at' (standard implementation) will be written back to 'dst'.
+  // As 'base' is required for the C-call, it must be reserved in case of a register clash.
+  Register saved_base = base;
+  if (base == dst) {
+    __ mr(tmp2, base);
+    saved_base = tmp2;
+  }
+
+  BarrierSetAssembler::load_at(masm, decorators, type, base, ind_or_offs, dst,
+                               tmp1, noreg, preservation_level, L_handle_null);
+
+  /* ==== Check whether pointer is dirty ==== */
+  Label skip_barrier;
+
+  // Load bad mask into scratch register.
+  __ ld(tmp1, (intptr_t) XThreadLocalData::address_bad_mask_offset(), R16_thread);
+
+  // The color bits of the to-be-tested pointer do not have to be equivalent to the 'bad_mask' testing bits.
+  // A pointer is classified as dirty if any of the color bits that also match the bad mask is set.
+  // Conversely, it follows that the logical AND of the bad mask and the pointer must be zero
+  // if the pointer is not dirty.
+  // Only dirty pointers must be processed by this barrier, so we can skip it in case the latter condition holds true.
+  __ and_(tmp1, tmp1, dst);
+  __ beq(CCR0, skip_barrier);
+
+  /* ==== Invoke barrier ==== */
+  int nbytes_save = 0;
+
+  const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
+  const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
+  const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
+
+  const bool preserve_R3 = dst != R3_ARG1;
+
+  if (needs_frame) {
+    if (preserve_gp_registers) {
+      nbytes_save = (preserve_fp_registers
+                     ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
+                     : MacroAssembler::num_volatile_gp_regs) * BytesPerWord;
+      nbytes_save -= preserve_R3 ? 0 : BytesPerWord;
+      __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers, preserve_R3);
+    }
+
+    __ save_LR_CR(tmp1);
+    __ push_frame_reg_args(nbytes_save, tmp1);
+  }
+
+  // Setup arguments
+  if (saved_base != R3_ARG1) {
+    __ mr_if_needed(R3_ARG1, dst);
+    __ add(R4_ARG2, ind_or_offs, saved_base);
+  } else if (dst != R4_ARG2) {
+    __ add(R4_ARG2, ind_or_offs, saved_base);
+    __ mr(R3_ARG1, dst);
+  } else {
+    __ add(R0, ind_or_offs, saved_base);
+    __ mr(R3_ARG1, dst);
+    __ mr(R4_ARG2, R0);
+  }
+
+  __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators));
+
+  Register result = R3_RET;
+  if (needs_frame) {
+    __ pop_frame();
+    __ restore_LR_CR(tmp1);
+
+    if (preserve_R3) {
+      __ mr(R0, R3_RET);
+      result = R0;
+    }
+
+    if (preserve_gp_registers) {
+      __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers, preserve_R3);
+    }
+  }
+  __ mr_if_needed(dst, result);
+
+  __ bind(skip_barrier);
+  __ block_comment("} load_at (zgc)");
+}
+
+#ifdef ASSERT
+// The Z store barrier only verifies the pointers it is operating on and is thus a sole debugging measure.
+void XBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                    Register base, RegisterOrConstant ind_or_offs, Register val,
+                                    Register tmp1, Register tmp2, Register tmp3,
+                                    MacroAssembler::PreservationLevel preservation_level) {
+  __ block_comment("store_at (zgc) {");
+
+  // If the 'val' register is 'noreg', the to-be-stored value is a null pointer.
+  if (is_reference_type(type) && val != noreg) {
+    __ ld(tmp1, in_bytes(XThreadLocalData::address_bad_mask_offset()), R16_thread);
+    __ and_(tmp1, tmp1, val);
+    __ asm_assert_eq("Detected dirty pointer on the heap in Z store barrier");
+  }
+
+  // Store value
+  BarrierSetAssembler::store_at(masm, decorators, type, base, ind_or_offs, val, tmp1, tmp2, tmp3, preservation_level);
+
+  __ block_comment("} store_at (zgc)");
+}
+#endif // ASSERT
+
+void XBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType component_type,
+                                              Register src, Register dst, Register count,
+                                              Register preserve1, Register preserve2) {
+  __ block_comment("arraycopy_prologue (zgc) {");
+
+  /* ==== Check whether a special gc barrier is required for this particular load ==== */
+  if (!is_reference_type(component_type)) {
+    return;
+  }
+
+  Label skip_barrier;
+
+  // Fast path: Array is of length zero
+  __ cmpdi(CCR0, count, 0);
+  __ beq(CCR0, skip_barrier);
+
+  /* ==== Ensure register sanity ==== */
+  Register tmp_R11 = R11_scratch1;
+
+  assert_different_registers(src, dst, count, tmp_R11, noreg);
+  if (preserve1 != noreg) {
+    // Not technically required, but unlikely being intended.
+    assert_different_registers(preserve1, preserve2);
+  }
+
+  /* ==== Invoke barrier (slowpath) ==== */
+  int nbytes_save = 0;
+
+  {
+    assert(!noreg->is_volatile(), "sanity");
+
+    if (preserve1->is_volatile()) {
+      __ std(preserve1, -BytesPerWord * ++nbytes_save, R1_SP);
+    }
+
+    if (preserve2->is_volatile() && preserve1 != preserve2) {
+      __ std(preserve2, -BytesPerWord * ++nbytes_save, R1_SP);
+    }
+
+    __ std(src, -BytesPerWord * ++nbytes_save, R1_SP);
+    __ std(dst, -BytesPerWord * ++nbytes_save, R1_SP);
+    __ std(count, -BytesPerWord * ++nbytes_save, R1_SP);
+
+    __ save_LR_CR(tmp_R11);
+    __ push_frame_reg_args(nbytes_save, tmp_R11);
+  }
+
+  // XBarrierSetRuntime::load_barrier_on_oop_array_addr(src, count)
+  if (count == R3_ARG1) {
+    if (src == R4_ARG2) {
+      // Arguments are provided in reverse order
+      __ mr(tmp_R11, count);
+      __ mr(R3_ARG1, src);
+      __ mr(R4_ARG2, tmp_R11);
+    } else {
+      __ mr(R4_ARG2, count);
+      __ mr(R3_ARG1, src);
+    }
+  } else {
+    __ mr_if_needed(R3_ARG1, src);
+    __ mr_if_needed(R4_ARG2, count);
+  }
+
+  __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_array_addr());
+
+  __ pop_frame();
+  __ restore_LR_CR(tmp_R11);
+
+  {
+    __ ld(count, -BytesPerWord * nbytes_save--, R1_SP);
+    __ ld(dst, -BytesPerWord * nbytes_save--, R1_SP);
+    __ ld(src, -BytesPerWord * nbytes_save--, R1_SP);
+
+    if (preserve2->is_volatile() && preserve1 != preserve2) {
+      __ ld(preserve2, -BytesPerWord * nbytes_save--, R1_SP);
+    }
+
+    if (preserve1->is_volatile()) {
+      __ ld(preserve1, -BytesPerWord * nbytes_save--, R1_SP);
+    }
+  }
+
+  __ bind(skip_barrier);
+
+  __ block_comment("} arraycopy_prologue (zgc)");
+}
+
+void XBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
+                                                         Register obj, Register tmp, Label& slowpath) {
+  __ block_comment("try_resolve_jobject_in_native (zgc) {");
+
+  assert_different_registers(jni_env, obj, tmp);
+
+  // Resolve the pointer using the standard implementation for weak tag handling and pointer verification.
+  BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
+
+  // Check whether pointer is dirty.
+  __ ld(tmp,
+        in_bytes(XThreadLocalData::address_bad_mask_offset() - JavaThread::jni_environment_offset()),
+        jni_env);
+
+  __ and_(tmp, obj, tmp);
+  __ bne(CCR0, slowpath);
+
+  __ block_comment("} try_resolve_jobject_in_native (zgc)");
+}
+
+#undef __
+
+#ifdef COMPILER1
+#define __ ce->masm()->
+
+// Code emitted by LIR node "LIR_OpXLoadBarrierTest" which in turn is emitted by XBarrierSetC1::load_barrier.
+// The actual compare and branch instructions are represented as stand-alone LIR nodes.
+void XBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
+                                                         LIR_Opr ref) const {
+  __ block_comment("load_barrier_test (zgc) {");
+
+  __ ld(R0, in_bytes(XThreadLocalData::address_bad_mask_offset()), R16_thread);
+  __ andr(R0, R0, ref->as_pointer_register());
+  __ cmpdi(CCR5 /* as mandated by LIR node */, R0, 0);
+
+  __ block_comment("} load_barrier_test (zgc)");
+}
+
+// Code emitted by code stub "XLoadBarrierStubC1" which in turn is emitted by XBarrierSetC1::load_barrier.
+// Invokes the runtime stub which is defined just below.
+void XBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
+                                                         XLoadBarrierStubC1* stub) const {
+  __ block_comment("c1_load_barrier_stub (zgc) {");
+
+  __ bind(*stub->entry());
+
+  /* ==== Determine relevant data registers and ensure register sanity ==== */
+  Register ref = stub->ref()->as_register();
+  Register ref_addr = noreg;
+
+  // Determine reference address
+  if (stub->tmp()->is_valid()) {
+    // 'tmp' register is given, so address might have an index or a displacement.
+    ce->leal(stub->ref_addr(), stub->tmp());
+    ref_addr = stub->tmp()->as_pointer_register();
+  } else {
+    // 'tmp' register is not given, so address must have neither an index nor a displacement.
+    // The address' base register is thus usable as-is.
+    assert(stub->ref_addr()->as_address_ptr()->disp() == 0, "illegal displacement");
+    assert(!stub->ref_addr()->as_address_ptr()->index()->is_valid(), "illegal index");
+
+    ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
+  }
+
+  assert_different_registers(ref, ref_addr, R0, noreg);
+
+  /* ==== Invoke stub ==== */
+  // Pass arguments via stack. The stack pointer will be bumped by the stub.
+  __ std(ref, (intptr_t) -1 * BytesPerWord, R1_SP);
+  __ std(ref_addr, (intptr_t) -2 * BytesPerWord, R1_SP);
+
+  __ load_const_optimized(R0, stub->runtime_stub());
+  __ call_stub(R0);
+
+  // The runtime stub passes the result via the R0 register, overriding the previously-loaded stub address.
+  __ mr_if_needed(ref, R0);
+  __ b(*stub->continuation());
+
+  __ block_comment("} c1_load_barrier_stub (zgc)");
+}
+
+#undef __
+#define __ sasm->
+
+// Code emitted by runtime code stub which in turn is emitted by XBarrierSetC1::generate_c1_runtime_stubs.
+void XBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
+                                                                 DecoratorSet decorators) const {
+  __ block_comment("c1_load_barrier_runtime_stub (zgc) {");
+
+  const int stack_parameters = 2;
+  const int nbytes_save = (MacroAssembler::num_volatile_regs + stack_parameters) * BytesPerWord;
+
+  __ save_volatile_gprs(R1_SP, -nbytes_save);
+  __ save_LR_CR(R0);
+
+  // Load arguments back again from the stack.
+  __ ld(R3_ARG1, (intptr_t) -1 * BytesPerWord, R1_SP); // ref
+  __ ld(R4_ARG2, (intptr_t) -2 * BytesPerWord, R1_SP); // ref_addr
+
+  __ push_frame_reg_args(nbytes_save, R0);
+
+  __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators));
+
+  __ verify_oop(R3_RET, "Bad pointer after barrier invocation");
+  __ mr(R0, R3_RET);
+
+  __ pop_frame();
+  __ restore_LR_CR(R3_RET);
+  __ restore_volatile_gprs(R1_SP, -nbytes_save);
+
+  __ blr();
+
+  __ block_comment("} c1_load_barrier_runtime_stub (zgc)");
+}
+
+#undef __
+#endif // COMPILER1
+
+#ifdef COMPILER2
+
+OptoReg::Name XBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) const {
+  if (!OptoReg::is_reg(opto_reg)) {
+    return OptoReg::Bad;
+  }
+
+  VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+  if ((vm_reg->is_Register() || vm_reg ->is_FloatRegister()) && (opto_reg & 1) != 0) {
+    return OptoReg::Bad;
+  }
+
+  return opto_reg;
+}
+
+#define __ _masm->
+
+class XSaveLiveRegisters {
+  MacroAssembler* _masm;
+  RegMask _reg_mask;
+  Register _result_reg;
+  int _frame_size;
+
+ public:
+  XSaveLiveRegisters(MacroAssembler *masm, XLoadBarrierStubC2 *stub)
+      : _masm(masm), _reg_mask(stub->live()), _result_reg(stub->ref()) {
+
+    const int register_save_size = iterate_over_register_mask(ACTION_COUNT_ONLY) * BytesPerWord;
+    _frame_size = align_up(register_save_size, frame::alignment_in_bytes)
+                  + frame::native_abi_reg_args_size;
+
+    __ save_LR_CR(R0);
+    __ push_frame(_frame_size, R0);
+
+    iterate_over_register_mask(ACTION_SAVE, _frame_size);
+  }
+
+  ~XSaveLiveRegisters() {
+    iterate_over_register_mask(ACTION_RESTORE, _frame_size);
+
+    __ addi(R1_SP, R1_SP, _frame_size);
+    __ restore_LR_CR(R0);
+  }
+
+ private:
+  enum IterationAction : int {
+    ACTION_SAVE,
+    ACTION_RESTORE,
+    ACTION_COUNT_ONLY
+  };
+
+  int iterate_over_register_mask(IterationAction action, int offset = 0) {
+    int reg_save_index = 0;
+    RegMaskIterator live_regs_iterator(_reg_mask);
+
+    while(live_regs_iterator.has_next()) {
+      const OptoReg::Name opto_reg = live_regs_iterator.next();
+
+      // Filter out stack slots (spilled registers, i.e., stack-allocated registers).
+      if (!OptoReg::is_reg(opto_reg)) {
+        continue;
+      }
+
+      const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+      if (vm_reg->is_Register()) {
+        Register std_reg = vm_reg->as_Register();
+
+        // '_result_reg' will hold the end result of the operation. Its content must thus not be preserved.
+        if (std_reg == _result_reg) {
+          continue;
+        }
+
+        if (std_reg->encoding() >= R2->encoding() && std_reg->encoding() <= R12->encoding()) {
+          reg_save_index++;
+
+          if (action == ACTION_SAVE) {
+            _masm->std(std_reg, offset - reg_save_index * BytesPerWord, R1_SP);
+          } else if (action == ACTION_RESTORE) {
+            _masm->ld(std_reg, offset - reg_save_index * BytesPerWord, R1_SP);
+          } else {
+            assert(action == ACTION_COUNT_ONLY, "Sanity");
+          }
+        }
+      } else if (vm_reg->is_FloatRegister()) {
+        FloatRegister fp_reg = vm_reg->as_FloatRegister();
+        if (fp_reg->encoding() >= F0->encoding() && fp_reg->encoding() <= F13->encoding()) {
+          reg_save_index++;
+
+          if (action == ACTION_SAVE) {
+            _masm->stfd(fp_reg, offset - reg_save_index * BytesPerWord, R1_SP);
+          } else if (action == ACTION_RESTORE) {
+            _masm->lfd(fp_reg, offset - reg_save_index * BytesPerWord, R1_SP);
+          } else {
+            assert(action == ACTION_COUNT_ONLY, "Sanity");
+          }
+        }
+      } else if (vm_reg->is_ConditionRegister()) {
+        // NOP. Conditions registers are covered by save_LR_CR
+      } else if (vm_reg->is_VectorSRegister()) {
+        assert(SuperwordUseVSX, "or should not reach here");
+        VectorSRegister vs_reg = vm_reg->as_VectorSRegister();
+        if (vs_reg->encoding() >= VSR32->encoding() && vs_reg->encoding() <= VSR51->encoding()) {
+          reg_save_index += 2;
+
+          Register spill_addr = R0;
+          if (action == ACTION_SAVE) {
+            _masm->addi(spill_addr, R1_SP, offset - reg_save_index * BytesPerWord);
+            _masm->stxvd2x(vs_reg, spill_addr);
+          } else if (action == ACTION_RESTORE) {
+            _masm->addi(spill_addr, R1_SP, offset - reg_save_index * BytesPerWord);
+            _masm->lxvd2x(vs_reg, spill_addr);
+          } else {
+            assert(action == ACTION_COUNT_ONLY, "Sanity");
+          }
+        }
+      } else {
+        if (vm_reg->is_SpecialRegister()) {
+          fatal("Special registers are unsupported. Found register %s", vm_reg->name());
+        } else {
+          fatal("Register type is not known");
+        }
+      }
+    }
+
+    return reg_save_index;
+  }
+};
+
+#undef __
+#define __ _masm->
+
+class XSetupArguments {
+  MacroAssembler* const _masm;
+  const Register        _ref;
+  const Address         _ref_addr;
+
+ public:
+  XSetupArguments(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
+      _masm(masm),
+      _ref(stub->ref()),
+      _ref_addr(stub->ref_addr()) {
+
+    // Desired register/argument configuration:
+    // _ref: R3_ARG1
+    // _ref_addr: R4_ARG2
+
+    // '_ref_addr' can be unspecified. In that case, the barrier will not heal the reference.
+    if (_ref_addr.base() == noreg) {
+      assert_different_registers(_ref, R0, noreg);
+
+      __ mr_if_needed(R3_ARG1, _ref);
+      __ li(R4_ARG2, 0);
+    } else {
+      assert_different_registers(_ref, _ref_addr.base(), R0, noreg);
+      assert(!_ref_addr.index()->is_valid(), "reference addresses must not contain an index component");
+
+      if (_ref != R4_ARG2) {
+        // Calculate address first as the address' base register might clash with R4_ARG2
+        __ addi(R4_ARG2, _ref_addr.base(), _ref_addr.disp());
+        __ mr_if_needed(R3_ARG1, _ref);
+      } else if (_ref_addr.base() != R3_ARG1) {
+        __ mr(R3_ARG1, _ref);
+        __ addi(R4_ARG2, _ref_addr.base(), _ref_addr.disp()); // Clobbering _ref
+      } else {
+        // Arguments are provided in inverse order (i.e. _ref == R4_ARG2, _ref_addr == R3_ARG1)
+        __ mr(R0, _ref);
+        __ addi(R4_ARG2, _ref_addr.base(), _ref_addr.disp());
+        __ mr(R3_ARG1, R0);
+      }
+    }
+  }
+};
+
+#undef __
+#define __ masm->
+
+void XBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, XLoadBarrierStubC2* stub) const {
+  __ block_comment("generate_c2_load_barrier_stub (zgc) {");
+
+  __ bind(*stub->entry());
+
+  Register ref = stub->ref();
+  Address ref_addr = stub->ref_addr();
+
+  assert_different_registers(ref, ref_addr.base());
+
+  {
+    XSaveLiveRegisters save_live_registers(masm, stub);
+    XSetupArguments setup_arguments(masm, stub);
+
+    __ call_VM_leaf(stub->slow_path());
+    __ mr_if_needed(ref, R3_RET);
+  }
+
+  __ b(*stub->continuation());
+
+  __ block_comment("} generate_c2_load_barrier_stub (zgc)");
+}
+
+#undef __
+#endif // COMPILER2
diff --git a/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.hpp
new file mode 100644
index 00000000000..14c338dd212
--- /dev/null
+++ b/src/hotspot/cpu/ppc/gc/x/xBarrierSetAssembler_ppc.hpp
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2022 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_PPC_GC_X_XBARRIERSETASSEMBLER_PPC_HPP
+#define CPU_PPC_GC_X_XBARRIERSETASSEMBLER_PPC_HPP
+
+#include "code/vmreg.hpp"
+#include "oops/accessDecorators.hpp"
+#ifdef COMPILER2
+#include "opto/optoreg.hpp"
+#endif // COMPILER2
+
+#ifdef COMPILER1
+class LIR_Assembler;
+class LIR_Opr;
+class StubAssembler;
+#endif // COMPILER1
+
+#ifdef COMPILER2
+class Node;
+#endif // COMPILER2
+
+#ifdef COMPILER1
+class XLoadBarrierStubC1;
+#endif // COMPILER1
+
+#ifdef COMPILER2
+class XLoadBarrierStubC2;
+#endif // COMPILER2
+
+class XBarrierSetAssembler : public XBarrierSetAssemblerBase {
+public:
+  virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                       Register base, RegisterOrConstant ind_or_offs, Register dst,
+                       Register tmp1, Register tmp2,
+                       MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null = NULL);
+
+#ifdef ASSERT
+  virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                        Register base, RegisterOrConstant ind_or_offs, Register val,
+                        Register tmp1, Register tmp2, Register tmp3,
+                        MacroAssembler::PreservationLevel preservation_level);
+#endif // ASSERT
+
+  virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
+                                  Register src, Register dst, Register count,
+                                  Register preserve1, Register preserve2);
+
+  virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
+                                             Register obj, Register tmp, Label& slowpath);
+
+  virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
+
+#ifdef COMPILER1
+  void generate_c1_load_barrier_test(LIR_Assembler* ce,
+                                     LIR_Opr ref) const;
+
+  void generate_c1_load_barrier_stub(LIR_Assembler* ce,
+                                     XLoadBarrierStubC1* stub) const;
+
+  void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
+                                             DecoratorSet decorators) const;
+#endif // COMPILER1
+
+#ifdef COMPILER2
+  OptoReg::Name refine_register(const Node* node, OptoReg::Name opto_reg) const;
+
+  void generate_c2_load_barrier_stub(MacroAssembler* masm, XLoadBarrierStubC2* stub) const;
+#endif // COMPILER2
+};
+
+#endif // CPU_PPC_GC_X_XBARRIERSETASSEMBLER_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/gc/z/zGlobals_ppc.cpp b/src/hotspot/cpu/ppc/gc/x/xGlobals_ppc.cpp
similarity index 97%
rename from src/hotspot/cpu/ppc/gc/z/zGlobals_ppc.cpp
rename to src/hotspot/cpu/ppc/gc/x/xGlobals_ppc.cpp
index fbf6d13dc8f..3218a765fc7 100644
--- a/src/hotspot/cpu/ppc/gc/z/zGlobals_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/x/xGlobals_ppc.cpp
@@ -25,7 +25,7 @@
 #include "precompiled.hpp"
 #include "gc/shared/gcLogPrecious.hpp"
 #include "gc/shared/gc_globals.hpp"
-#include "gc/z/zGlobals.hpp"
+#include "gc/x/xGlobals.hpp"
 #include "runtime/globals.hpp"
 #include "runtime/os.hpp"
 #include "utilities/globalDefinitions.hpp"
@@ -154,7 +154,7 @@ static unsigned int probe_valid_max_address_bit(size_t init_bit, size_t min_bit)
     // It should thus be a "close enough" approximation to the real virtual memory address space limit.
     //
     // This recovery strategy is only applied in production builds.
-    // In debug builds, an assertion in 'ZPlatformAddressOffsetBits' will bail out the VM to indicate that
+    // In debug builds, an assertion in 'XPlatformAddressOffsetBits' will bail out the VM to indicate that
     // the assumed address space is no longer up-to-date.
     if (last_allocatable_address != MAP_FAILED) {
       const unsigned int bitpos = BitsPerSize_t - count_leading_zeros((size_t) last_allocatable_address) - 1;
@@ -184,7 +184,7 @@ static unsigned int probe_valid_max_address_bit(size_t init_bit, size_t min_bit)
 #endif // LINUX
 }
 
-size_t ZPlatformAddressOffsetBits() {
+size_t XPlatformAddressOffsetBits() {
   const static unsigned int valid_max_address_offset_bits =
       probe_valid_max_address_bit(DEFAULT_MAX_ADDRESS_BIT, MINIMUM_MAX_ADDRESS_BIT) + 1;
   assert(valid_max_address_offset_bits >= MINIMUM_MAX_ADDRESS_BIT,
@@ -192,12 +192,12 @@ size_t ZPlatformAddressOffsetBits() {
 
   const size_t max_address_offset_bits = valid_max_address_offset_bits - 3;
   const size_t min_address_offset_bits = max_address_offset_bits - 2;
-  const size_t address_offset = round_up_power_of_2(MaxHeapSize * ZVirtualToPhysicalRatio);
+  const size_t address_offset = round_up_power_of_2(MaxHeapSize * XVirtualToPhysicalRatio);
   const size_t address_offset_bits = log2i_exact(address_offset);
 
   return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
 }
 
-size_t ZPlatformAddressMetadataShift() {
-  return ZPlatformAddressOffsetBits();
+size_t XPlatformAddressMetadataShift() {
+  return XPlatformAddressOffsetBits();
 }
diff --git a/src/hotspot/cpu/ppc/gc/x/xGlobals_ppc.hpp b/src/hotspot/cpu/ppc/gc/x/xGlobals_ppc.hpp
new file mode 100644
index 00000000000..be88b05b02a
--- /dev/null
+++ b/src/hotspot/cpu/ppc/gc/x/xGlobals_ppc.hpp
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_PPC_GC_X_XGLOBALS_PPC_HPP
+#define CPU_PPC_GC_X_XGLOBALS_PPC_HPP
+
+#include "globalDefinitions_ppc.hpp"
+
+const size_t XPlatformHeapViews        = 3;
+const size_t XPlatformCacheLineSize    = DEFAULT_CACHE_LINE_SIZE;
+
+size_t XPlatformAddressOffsetBits();
+size_t XPlatformAddressMetadataShift();
+
+#endif // CPU_PPC_GC_X_XGLOBALS_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/gc/x/x_ppc.ad b/src/hotspot/cpu/ppc/gc/x/x_ppc.ad
new file mode 100644
index 00000000000..dd46b46a3a3
--- /dev/null
+++ b/src/hotspot/cpu/ppc/gc/x/x_ppc.ad
@@ -0,0 +1,298 @@
+//
+// Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2021 SAP SE. All rights reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+
+source_hpp %{
+
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/c2/xBarrierSetC2.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+
+%}
+
+source %{
+
+static void x_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref,
+                           Register tmp, uint8_t barrier_data) {
+  if (barrier_data == XLoadBarrierElided) {
+    return;
+  }
+
+  XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
+  __ ld(tmp, in_bytes(XThreadLocalData::address_bad_mask_offset()), R16_thread);
+  __ and_(tmp, tmp, ref);
+  __ bne_far(CCR0, *stub->entry(), MacroAssembler::bc_far_optimize_on_relocate);
+  __ bind(*stub->continuation());
+}
+
+static void x_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref,
+                                     Register tmp) {
+  XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, XLoadBarrierStrong);
+  __ b(*stub->entry());
+  __ bind(*stub->continuation());
+}
+
+static void x_compare_and_swap(MacroAssembler& _masm, const MachNode* node,
+                              Register res, Register mem, Register oldval, Register newval,
+                              Register tmp_xchg, Register tmp_mask,
+                              bool weak, bool acquire) {
+  // z-specific load barrier requires strong CAS operations.
+  // Weak CAS operations are thus only emitted if the barrier is elided.
+  __ cmpxchgd(CCR0, tmp_xchg, oldval, newval, mem,
+              MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), res, NULL, true,
+              weak && node->barrier_data() == XLoadBarrierElided);
+
+  if (node->barrier_data() != XLoadBarrierElided) {
+    Label skip_barrier;
+
+    __ ld(tmp_mask, in_bytes(XThreadLocalData::address_bad_mask_offset()), R16_thread);
+    __ and_(tmp_mask, tmp_mask, tmp_xchg);
+    __ beq(CCR0, skip_barrier);
+
+    // CAS must have failed because pointer in memory is bad.
+    x_load_barrier_slow_path(_masm, node, Address(mem), tmp_xchg, res /* used as tmp */);
+
+    __ cmpxchgd(CCR0, tmp_xchg, oldval, newval, mem,
+                MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), res, NULL, true, weak);
+
+    __ bind(skip_barrier);
+  }
+
+  if (acquire) {
+    if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
+      // Uses the isync instruction as an acquire barrier.
+      // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync).
+      __ isync();
+    } else {
+      __ sync();
+    }
+  }
+}
+
+static void x_compare_and_exchange(MacroAssembler& _masm, const MachNode* node,
+                                   Register res, Register mem, Register oldval, Register newval, Register tmp,
+                                   bool weak, bool acquire) {
+  // z-specific load barrier requires strong CAS operations.
+  // Weak CAS operations are thus only emitted if the barrier is elided.
+  __ cmpxchgd(CCR0, res, oldval, newval, mem,
+              MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), noreg, NULL, true,
+              weak && node->barrier_data() == XLoadBarrierElided);
+
+  if (node->barrier_data() != XLoadBarrierElided) {
+    Label skip_barrier;
+    __ ld(tmp, in_bytes(XThreadLocalData::address_bad_mask_offset()), R16_thread);
+    __ and_(tmp, tmp, res);
+    __ beq(CCR0, skip_barrier);
+
+    x_load_barrier_slow_path(_masm, node, Address(mem), res, tmp);
+
+    __ cmpxchgd(CCR0, res, oldval, newval, mem,
+                MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), noreg, NULL, true, weak);
+
+    __ bind(skip_barrier);
+  }
+
+  if (acquire) {
+    if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
+      // Uses the isync instruction as an acquire barrier.
+      // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync).
+      __ isync();
+    } else {
+      __ sync();
+    }
+  }
+}
+
+%}
+
+instruct xLoadP(iRegPdst dst, memoryAlg4 mem, iRegPdst tmp, flagsRegCR0 cr0)
+%{
+  match(Set dst (LoadP mem));
+  effect(TEMP_DEF dst, TEMP tmp, KILL cr0);
+  ins_cost(MEMORY_REF_COST);
+
+  predicate((UseZGC && !ZGenerational && n->as_Load()->barrier_data() != 0)
+            && (n->as_Load()->is_unordered() || followed_by_acquire(n)));
+
+  format %{ "LD $dst, $mem" %}
+  ins_encode %{
+    assert($mem$$index == 0, "sanity");
+    __ ld($dst$$Register, $mem$$disp, $mem$$base$$Register);
+    x_load_barrier(_masm, this, Address($mem$$base$$Register, $mem$$disp), $dst$$Register, $tmp$$Register, barrier_data());
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
+// Load Pointer Volatile
+instruct xLoadP_acq(iRegPdst dst, memoryAlg4 mem, iRegPdst tmp, flagsRegCR0 cr0)
+%{
+  match(Set dst (LoadP mem));
+  effect(TEMP_DEF dst, TEMP tmp, KILL cr0);
+  ins_cost(3 * MEMORY_REF_COST);
+
+  // Predicate on instruction order is implicitly present due to the predicate of the cheaper zLoadP operation
+  predicate(UseZGC && !ZGenerational && n->as_Load()->barrier_data() != 0);
+
+  format %{ "LD acq $dst, $mem" %}
+  ins_encode %{
+    __ ld($dst$$Register, $mem$$disp, $mem$$base$$Register);
+    x_load_barrier(_masm, this, Address($mem$$base$$Register, $mem$$disp), $dst$$Register, $tmp$$Register, barrier_data());
+
+    // Uses the isync instruction as an acquire barrier.
+    // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync).
+    __ isync();
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
+instruct xCompareAndSwapP(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
+                                iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
+  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+  effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
+
+  predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong)
+            && (((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*) n)->order() != MemNode::seqcst));
+
+  format %{ "CMPXCHG $res, $mem, $oldval, $newval; as bool; ptr" %}
+  ins_encode %{
+    x_compare_and_swap(_masm, this,
+                                $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
+                                $tmp_xchg$$Register, $tmp_mask$$Register,
+                                false /* weak */, false /* acquire */);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
+instruct xCompareAndSwapP_acq(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
+                                    iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
+  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+  effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
+
+  predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong)
+            && (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*) n)->order() == MemNode::seqcst));
+
+  format %{ "CMPXCHG acq $res, $mem, $oldval, $newval; as bool; ptr" %}
+  ins_encode %{
+    x_compare_and_swap(_masm, this,
+                                $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
+                                $tmp_xchg$$Register, $tmp_mask$$Register,
+                                false /* weak */, true /* acquire */);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
+instruct xCompareAndSwapPWeak(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
+                                    iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
+  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+  effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
+
+  predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong)
+            && ((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*) n)->order() != MemNode::seqcst);
+
+  format %{ "weak CMPXCHG $res, $mem, $oldval, $newval; as bool; ptr" %}
+  ins_encode %{
+    x_compare_and_swap(_masm, this,
+                                $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
+                                $tmp_xchg$$Register, $tmp_mask$$Register,
+                                true /* weak */, false /* acquire */);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
+instruct xCompareAndSwapPWeak_acq(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
+                                        iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
+  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+  effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
+
+  predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong)
+            && (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*) n)->order() == MemNode::seqcst));
+
+  format %{ "weak CMPXCHG acq $res, $mem, $oldval, $newval; as bool; ptr" %}
+  ins_encode %{
+    x_compare_and_swap(_masm, this,
+                                $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
+                                $tmp_xchg$$Register, $tmp_mask$$Register,
+                                true /* weak */, true /* acquire */);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
+instruct xCompareAndExchangeP(iRegPdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
+                              iRegPdst tmp, flagsRegCR0 cr0) %{
+  match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+  effect(TEMP_DEF res, TEMP tmp, KILL cr0);
+
+  predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong)
+            && (
+              ((CompareAndSwapNode*)n)->order() != MemNode::acquire
+              && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst
+            ));
+
+  format %{ "CMPXCHG $res, $mem, $oldval, $newval; as ptr; ptr" %}
+  ins_encode %{
+    x_compare_and_exchange(_masm, this,
+                                    $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, $tmp$$Register,
+                                    false /* weak */, false /* acquire */);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
+instruct xCompareAndExchangeP_acq(iRegPdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
+                                        iRegPdst tmp, flagsRegCR0 cr0) %{
+  match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+  effect(TEMP_DEF res, TEMP tmp, KILL cr0);
+
+  predicate((UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong)
+            && (
+              ((CompareAndSwapNode*)n)->order() == MemNode::acquire
+              || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst
+            ));
+
+  format %{ "CMPXCHG acq $res, $mem, $oldval, $newval; as ptr; ptr" %}
+  ins_encode %{
+    x_compare_and_exchange(_masm, this,
+                                    $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, $tmp$$Register,
+                                    false /* weak */, true /* acquire */);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
+instruct xGetAndSetP(iRegPdst res, iRegPdst mem, iRegPsrc newval, iRegPdst tmp, flagsRegCR0 cr0) %{
+  match(Set res (GetAndSetP mem newval));
+  effect(TEMP_DEF res, TEMP tmp, KILL cr0);
+
+  predicate(UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() != 0);
+
+  format %{ "GetAndSetP $res, $mem, $newval" %}
+  ins_encode %{
+    __ getandsetd($res$$Register, $newval$$Register, $mem$$Register, MacroAssembler::cmpxchgx_hint_atomic_update());
+    x_load_barrier(_masm, this, Address(noreg, (intptr_t) 0), $res$$Register, $tmp$$Register, barrier_data());
+
+    if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
+      __ isync();
+    } else {
+      __ sync();
+    }
+  %}
+  ins_pipe(pipe_class_default);
+%}
diff --git a/src/hotspot/cpu/ppc/gc/z/zAddress_ppc.cpp b/src/hotspot/cpu/ppc/gc/z/zAddress_ppc.cpp
new file mode 100644
index 00000000000..136fd7a8ad1
--- /dev/null
+++ b/src/hotspot/cpu/ppc/gc/z/zAddress_ppc.cpp
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/os.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+#ifdef LINUX
+#include <sys/mman.h>
+#endif // LINUX
+
+// Default value if probing is not implemented for a certain platform: 128TB
+static const size_t DEFAULT_MAX_ADDRESS_BIT = 47;
+// Minimum value returned, if probing fails: 64GB
+static const size_t MINIMUM_MAX_ADDRESS_BIT = 36;
+
+static size_t probe_valid_max_address_bit() {
+#ifdef LINUX
+  size_t max_address_bit = 0;
+  const size_t page_size = os::vm_page_size();
+  for (size_t i = DEFAULT_MAX_ADDRESS_BIT; i > MINIMUM_MAX_ADDRESS_BIT; --i) {
+    const uintptr_t base_addr = ((uintptr_t) 1U) << i;
+    if (msync((void*)base_addr, page_size, MS_ASYNC) == 0) {
+      // msync suceeded, the address is valid, and maybe even already mapped.
+      max_address_bit = i;
+      break;
+    }
+    if (errno != ENOMEM) {
+      // Some error occured. This should never happen, but msync
+      // has some undefined behavior, hence ignore this bit.
+#ifdef ASSERT
+      fatal("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
+#else // ASSERT
+      log_warning_p(gc)("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
+#endif // ASSERT
+      continue;
+    }
+    // Since msync failed with ENOMEM, the page might not be mapped.
+    // Try to map it, to see if the address is valid.
+    void* const result_addr = mmap((void*) base_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
+    if (result_addr != MAP_FAILED) {
+      munmap(result_addr, page_size);
+    }
+    if ((uintptr_t) result_addr == base_addr) {
+      // address is valid
+      max_address_bit = i;
+      break;
+    }
+  }
+  if (max_address_bit == 0) {
+    // probing failed, allocate a very high page and take that bit as the maximum
+    const uintptr_t high_addr = ((uintptr_t) 1U) << DEFAULT_MAX_ADDRESS_BIT;
+    void* const result_addr = mmap((void*) high_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
+    if (result_addr != MAP_FAILED) {
+      max_address_bit = BitsPerSize_t - count_leading_zeros((size_t) result_addr) - 1;
+      munmap(result_addr, page_size);
+    }
+  }
+  log_info_p(gc, init)("Probing address space for the highest valid bit: " SIZE_FORMAT, max_address_bit);
+  return MAX2(max_address_bit, MINIMUM_MAX_ADDRESS_BIT);
+#else // LINUX
+  return DEFAULT_MAX_ADDRESS_BIT;
+#endif // LINUX
+}
+
+size_t ZPlatformAddressOffsetBits() {
+  const static size_t valid_max_address_offset_bits = probe_valid_max_address_bit() + 1;
+  const size_t max_address_offset_bits = valid_max_address_offset_bits - 3;
+  const size_t min_address_offset_bits = max_address_offset_bits - 2;
+  const size_t address_offset = round_up_power_of_2(MaxHeapSize * ZVirtualToPhysicalRatio);
+  const size_t address_offset_bits = log2i_exact(address_offset);
+  return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
+}
+
+size_t ZPlatformAddressHeapBaseShift() {
+  return ZPlatformAddressOffsetBits();
+}
+
+void ZGlobalsPointers::pd_set_good_masks() {
+}
diff --git a/src/hotspot/cpu/ppc/gc/z/zAddress_ppc.hpp b/src/hotspot/cpu/ppc/gc/z/zAddress_ppc.hpp
new file mode 100644
index 00000000000..ffaadca4d82
--- /dev/null
+++ b/src/hotspot/cpu/ppc/gc/z/zAddress_ppc.hpp
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_PPC_GC_Z_ZADDRESS_PPC_HPP
+#define CPU_PPC_GC_Z_ZADDRESS_PPC_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+const size_t ZPointerLoadShift = 16;
+
+size_t ZPlatformAddressOffsetBits();
+size_t ZPlatformAddressHeapBaseShift();
+
+#endif // CPU_PPC_GC_Z_ZADDRESS_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/gc/z/zAddress_ppc.inline.hpp b/src/hotspot/cpu/ppc/gc/z/zAddress_ppc.inline.hpp
new file mode 100644
index 00000000000..438f4f5663c
--- /dev/null
+++ b/src/hotspot/cpu/ppc/gc/z/zAddress_ppc.inline.hpp
@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_PPC_GC_Z_ZADDRESS_PPC_INLINE_HPP
+#define CPU_PPC_GC_Z_ZADDRESS_PPC_INLINE_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+inline uintptr_t ZPointer::remap_bits(uintptr_t colored) {
+  return colored & ZPointerRemappedMask;
+}
+
+inline constexpr int ZPointer::load_shift_lookup(uintptr_t value) {
+  return ZPointerLoadShift;
+}
+
+#endif // CPU_PPC_GC_Z_ZADDRESS_PPC_INLINE_HPP
diff --git a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
index 84042505089..9ca0ee0ecf6 100644
--- a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
+++ b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.cpp
@@ -22,11 +22,12 @@
  * questions.
  */
 
-#include "asm/register.hpp"
 #include "precompiled.hpp"
 #include "asm/macroAssembler.inline.hpp"
+#include "asm/register.hpp"
 #include "code/codeBlob.hpp"
 #include "code/vmreg.inline.hpp"
+#include "gc/z/zAddress.hpp"
 #include "gc/z/zBarrier.inline.hpp"
 #include "gc/z/zBarrierSet.hpp"
 #include "gc/z/zBarrierSetAssembler.hpp"
@@ -34,6 +35,7 @@
 #include "gc/z/zThreadLocalData.hpp"
 #include "memory/resourceArea.hpp"
 #include "register_ppc.hpp"
+#include "runtime/jniHandles.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/macros.hpp"
@@ -44,11 +46,76 @@
 #endif // COMPILER1
 #ifdef COMPILER2
 #include "gc/z/c2/zBarrierSetC2.hpp"
+#include "opto/output.hpp"
 #endif // COMPILER2
 
 #undef __
 #define __ masm->
 
+// Helper for saving and restoring registers across a runtime call that does
+// not have any live vector registers.
+class ZRuntimeCallSpill {
+  MacroAssembler* _masm;
+  Register _result;
+  bool _needs_frame, _preserve_gp_registers, _preserve_fp_registers;
+  int _nbytes_save;
+
+  void save() {
+    MacroAssembler* masm = _masm;
+
+    if (_needs_frame) {
+      if (_preserve_gp_registers) {
+        bool preserve_R3 = _result != R3_ARG1;
+        _nbytes_save = (MacroAssembler::num_volatile_gp_regs
+                        + (_preserve_fp_registers ? MacroAssembler::num_volatile_fp_regs : 0)
+                        - (preserve_R3 ? 0 : 1)
+                       ) * BytesPerWord;
+        __ save_volatile_gprs(R1_SP, -_nbytes_save, _preserve_fp_registers, preserve_R3);
+      }
+
+      __ save_LR_CR(R0);
+      __ push_frame_reg_args(_nbytes_save, R0);
+    }
+  }
+
+  void restore() {
+    MacroAssembler* masm = _masm;
+
+    Register result = R3_RET;
+    if (_needs_frame) {
+      __ pop_frame();
+      __ restore_LR_CR(R0);
+
+      if (_preserve_gp_registers) {
+        bool restore_R3 = _result != R3_ARG1;
+        if (restore_R3 && _result != noreg) {
+          __ mr(R0, R3_RET);
+          result = R0;
+        }
+        __ restore_volatile_gprs(R1_SP, -_nbytes_save, _preserve_fp_registers, restore_R3);
+      }
+    }
+    if (_result != noreg) {
+      __ mr_if_needed(_result, result);
+    }
+  }
+
+public:
+  ZRuntimeCallSpill(MacroAssembler* masm, Register result, MacroAssembler::PreservationLevel preservation_level)
+    : _masm(masm),
+      _result(result),
+      _needs_frame(preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR),
+      _preserve_gp_registers(preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS),
+      _preserve_fp_registers(preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS),
+      _nbytes_save(0) {
+    save();
+  }
+  ~ZRuntimeCallSpill() {
+    restore();
+  }
+};
+
+
 void ZBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                                    Register base, RegisterOrConstant ind_or_offs, Register dst,
                                    Register tmp1, Register tmp2,
@@ -81,14 +148,21 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators
     saved_base = tmp2;
   }
 
-  BarrierSetAssembler::load_at(masm, decorators, type, base, ind_or_offs, dst,
-                               tmp1, noreg, preservation_level, L_handle_null);
+  __ ld(dst, ind_or_offs, base);
 
   /* ==== Check whether pointer is dirty ==== */
-  Label skip_barrier;
+  Label done, uncolor;
+
+  const bool on_non_strong =
+    (decorators & ON_WEAK_OOP_REF) != 0 ||
+    (decorators & ON_PHANTOM_OOP_REF) != 0;
 
   // Load bad mask into scratch register.
-  __ ld(tmp1, (intptr_t) ZThreadLocalData::address_bad_mask_offset(), R16_thread);
+  if (on_non_strong) {
+    __ ld(tmp1, in_bytes(ZThreadLocalData::mark_bad_mask_offset()), R16_thread);
+  } else {
+    __ ld(tmp1, in_bytes(ZThreadLocalData::load_bad_mask_offset()), R16_thread);
+  }
 
   // The color bits of the to-be-tested pointer do not have to be equivalent to the 'bad_mask' testing bits.
   // A pointer is classified as dirty if any of the color bits that also match the bad mask is set.
@@ -96,66 +170,195 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet decorators
   // if the pointer is not dirty.
   // Only dirty pointers must be processed by this barrier, so we can skip it in case the latter condition holds true.
   __ and_(tmp1, tmp1, dst);
-  __ beq(CCR0, skip_barrier);
+  __ beq(CCR0, uncolor);
 
   /* ==== Invoke barrier ==== */
-  int nbytes_save = 0;
+  {
+    ZRuntimeCallSpill rcs(masm, dst, preservation_level);
 
-  const bool needs_frame = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR;
-  const bool preserve_gp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS;
-  const bool preserve_fp_registers = preservation_level >= MacroAssembler::PRESERVATION_FRAME_LR_GP_FP_REGS;
-
-  const bool preserve_R3 = dst != R3_ARG1;
-
-  if (needs_frame) {
-    if (preserve_gp_registers) {
-      nbytes_save = (preserve_fp_registers
-                     ? MacroAssembler::num_volatile_gp_regs + MacroAssembler::num_volatile_fp_regs
-                     : MacroAssembler::num_volatile_gp_regs) * BytesPerWord;
-      nbytes_save -= preserve_R3 ? 0 : BytesPerWord;
-      __ save_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers, preserve_R3);
+    // Setup arguments
+    if (saved_base != R3_ARG1 && ind_or_offs.register_or_noreg() != R3_ARG1) {
+      __ mr_if_needed(R3_ARG1, dst);
+      __ add(R4_ARG2, ind_or_offs, saved_base);
+    } else if (dst != R4_ARG2) {
+      __ add(R4_ARG2, ind_or_offs, saved_base);
+      __ mr(R3_ARG1, dst);
+    } else {
+      __ add(R0, ind_or_offs, saved_base);
+      __ mr(R3_ARG1, dst);
+      __ mr(R4_ARG2, R0);
     }
 
-    __ save_LR_CR(tmp1);
-    __ push_frame_reg_args(nbytes_save, tmp1);
+    __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators));
   }
 
-  // Setup arguments
-  if (saved_base != R3_ARG1) {
-    __ mr_if_needed(R3_ARG1, dst);
-    __ add(R4_ARG2, ind_or_offs, saved_base);
-  } else if (dst != R4_ARG2) {
-    __ add(R4_ARG2, ind_or_offs, saved_base);
-    __ mr(R3_ARG1, dst);
+  // Slow-path has already uncolored
+  if (L_handle_null != nullptr) {
+    __ cmpdi(CCR0, dst, 0);
+    __ beq(CCR0, *L_handle_null);
+  }
+  __ b(done);
+
+  __ bind(uncolor);
+  if (L_handle_null == nullptr) {
+    __ srdi(dst, dst, ZPointerLoadShift);
   } else {
-    __ add(R0, ind_or_offs, saved_base);
-    __ mr(R3_ARG1, dst);
-    __ mr(R4_ARG2, R0);
+    __ srdi_(dst, dst, ZPointerLoadShift);
+    __ beq(CCR0, *L_handle_null);
   }
 
-  __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators));
-
-  Register result = R3_RET;
-  if (needs_frame) {
-    __ pop_frame();
-    __ restore_LR_CR(tmp1);
-
-    if (preserve_R3) {
-      __ mr(R0, R3_RET);
-      result = R0;
-    }
-
-    if (preserve_gp_registers) {
-      __ restore_volatile_gprs(R1_SP, -nbytes_save, preserve_fp_registers, preserve_R3);
-    }
-  }
-  __ mr_if_needed(dst, result);
-
-  __ bind(skip_barrier);
+  __ bind(done);
   __ block_comment("} load_at (zgc)");
 }
 
-#ifdef ASSERT
+static void load_least_significant_16_oop_bits(MacroAssembler* masm, Register dst, RegisterOrConstant ind_or_offs, Register base) {
+  assert_different_registers(dst, base);
+#ifndef VM_LITTLE_ENDIAN
+  const int BE_offset = 6;
+  if (ind_or_offs.is_register()) {
+    __ addi(dst, ind_or_offs.as_register(), BE_offset);
+    __ lhzx(dst, base, dst);
+  } else {
+    __ lhz(dst, ind_or_offs.as_constant() + BE_offset, base);
+  }
+#else
+  __ lhz(dst, ind_or_offs, base);
+#endif
+}
+
+static void emit_store_fast_path_check(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, bool is_atomic, Label& medium_path) {
+  if (is_atomic) {
+    assert(ZPointerLoadShift + LogMinObjAlignmentInBytes >= 16, "or replace following code");
+    load_least_significant_16_oop_bits(masm, R0, ind_or_offs, base);
+    // Atomic operations must ensure that the contents of memory are store-good before
+    // an atomic operation can execute.
+    // A not relocatable object could have spurious raw null pointers in its fields after
+    // getting promoted to the old generation.
+    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBits);
+    __ cmplwi(CCR0, R0, barrier_Relocation::unpatched);
+  } else {
+    __ ld(R0, ind_or_offs, base);
+    // Stores on relocatable objects never need to deal with raw null pointers in fields.
+    // Raw null pointers may only exist in the young generation, as they get pruned when
+    // the object is relocated to old. And no pre-write barrier needs to perform any action
+    // in the young generation.
+    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreBadMask);
+    __ andi_(R0, R0, barrier_Relocation::unpatched);
+  }
+  __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CCR0, Assembler::equal), medium_path);
+}
+
+void ZBarrierSetAssembler::store_barrier_fast(MacroAssembler* masm,
+                                              Register ref_base,
+                                              RegisterOrConstant ind_or_offset,
+                                              Register rnew_zaddress,
+                                              Register rnew_zpointer,
+                                              bool in_nmethod,
+                                              bool is_atomic,
+                                              Label& medium_path,
+                                              Label& medium_path_continuation) const {
+  assert_different_registers(ref_base, rnew_zpointer);
+  assert_different_registers(ind_or_offset.register_or_noreg(), rnew_zpointer);
+  assert_different_registers(rnew_zaddress, rnew_zpointer);
+
+  if (in_nmethod) {
+    emit_store_fast_path_check(masm, ref_base, ind_or_offset, is_atomic, medium_path);
+    __ bind(medium_path_continuation);
+    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBits);
+    __ li(rnew_zpointer, barrier_Relocation::unpatched); // Load color bits.
+    if (rnew_zaddress == noreg) { // noreg encodes null.
+      if (ZPointerLoadShift >= 16) {
+        __ rldicl(rnew_zpointer, rnew_zpointer, 0, 64 - ZPointerLoadShift); // Clear sign extension from li.
+      }
+    }
+  } else {
+    __ ld(R0, ind_or_offset, ref_base);
+    __ ld(rnew_zpointer, in_bytes(ZThreadLocalData::store_bad_mask_offset()), R16_thread);
+    __ and_(R0, R0, rnew_zpointer);
+    __ bne(CCR0, medium_path);
+    __ bind(medium_path_continuation);
+    __ ld(rnew_zpointer, in_bytes(ZThreadLocalData::store_good_mask_offset()), R16_thread);
+  }
+  if (rnew_zaddress != noreg) { // noreg encodes null.
+    __ rldimi(rnew_zpointer, rnew_zaddress, ZPointerLoadShift, 0); // Insert shifted pointer.
+  }
+}
+
+static void store_barrier_buffer_add(MacroAssembler* masm,
+                                     Register ref_base,
+                                     RegisterOrConstant ind_or_offs,
+                                     Register tmp1,
+                                     Label& slow_path) {
+  __ ld(tmp1, in_bytes(ZThreadLocalData::store_barrier_buffer_offset()), R16_thread);
+
+  // Combined pointer bump and check if the buffer is disabled or full
+  __ ld(R0, in_bytes(ZStoreBarrierBuffer::current_offset()), tmp1);
+  __ addic_(R0, R0, -(int)sizeof(ZStoreBarrierEntry));
+  __ blt(CCR0, slow_path);
+  __ std(R0, in_bytes(ZStoreBarrierBuffer::current_offset()), tmp1);
+
+  // Entry is at ZStoreBarrierBuffer (tmp1) + buffer_offset + scaled index (R0)
+  __ add(tmp1, tmp1, R0);
+
+  // Compute and log the store address
+  Register store_addr = ref_base;
+  if (!ind_or_offs.is_constant() || ind_or_offs.as_constant() != 0) {
+    __ add(R0, ind_or_offs, ref_base);
+    store_addr = R0;
+  }
+  __ std(store_addr, in_bytes(ZStoreBarrierBuffer::buffer_offset()) + in_bytes(ZStoreBarrierEntry::p_offset()), tmp1);
+
+  // Load and log the prev value
+  __ ld(R0, ind_or_offs, ref_base);
+  __ std(R0, in_bytes(ZStoreBarrierBuffer::buffer_offset()) + in_bytes(ZStoreBarrierEntry::prev_offset()), tmp1);
+}
+
+void ZBarrierSetAssembler::store_barrier_medium(MacroAssembler* masm,
+                                                Register ref_base,
+                                                RegisterOrConstant ind_or_offs,
+                                                Register tmp,
+                                                bool is_atomic,
+                                                Label& medium_path_continuation,
+                                                Label& slow_path) const {
+  assert_different_registers(ref_base, tmp, R0);
+
+  // The reason to end up in the medium path is that the pre-value was not 'good'.
+
+  if (is_atomic) {
+    // Atomic accesses can get to the medium fast path because the value was a
+    // raw null value. If it was not null, then there is no doubt we need to take a slow path.
+    __ ld(tmp, ind_or_offs, ref_base);
+    __ cmpdi(CCR0, tmp, 0);
+    __ bne(CCR0, slow_path);
+
+    // If we get this far, we know there is a young raw null value in the field.
+    // Try to self-heal null values for atomic accesses
+    bool need_restore = false;
+    if (!ind_or_offs.is_constant() || ind_or_offs.as_constant() != 0) {
+      __ add(ref_base, ind_or_offs, ref_base);
+      need_restore = true;
+    }
+    __ ld(R0, in_bytes(ZThreadLocalData::store_good_mask_offset()), R16_thread);
+    __ cmpxchgd(CCR0, tmp, (intptr_t)0, R0, ref_base,
+                MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update());
+    if (need_restore) {
+      __ subf(ref_base, ind_or_offs, ref_base);
+    }
+    __ bne(CCR0, slow_path);
+  } else {
+    // A non-atomic relocatable object won't get to the medium fast path due to a
+    // raw null in the young generation. We only get here because the field is bad.
+    // In this path we don't need any self healing, so we can avoid a runtime call
+    // most of the time by buffering the store barrier to be applied lazily.
+    store_barrier_buffer_add(masm,
+                             ref_base,
+                             ind_or_offs,
+                             tmp,
+                             slow_path);
+  }
+  __ b(medium_path_continuation);
+}
+
 // The Z store barrier only verifies the pointers it is operating on and is thus a sole debugging measure.
 void ZBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                                     Register base, RegisterOrConstant ind_or_offs, Register val,
@@ -163,124 +366,267 @@ void ZBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorator
                                     MacroAssembler::PreservationLevel preservation_level) {
   __ block_comment("store_at (zgc) {");
 
-  // If the 'val' register is 'noreg', the to-be-stored value is a null pointer.
-  if (is_reference_type(type) && val != noreg) {
-    __ ld(tmp1, in_bytes(ZThreadLocalData::address_bad_mask_offset()), R16_thread);
-    __ and_(tmp1, tmp1, val);
-    __ asm_assert_eq("Detected dirty pointer on the heap in Z store barrier");
-  }
+  bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 
-  // Store value
-  BarrierSetAssembler::store_at(masm, decorators, type, base, ind_or_offs, val, tmp1, tmp2, tmp3, preservation_level);
+  if (is_reference_type(type)) {
+    assert_different_registers(base, val, tmp1, tmp2, tmp3);
+
+    if (dest_uninitialized) {
+      // tmp1 = (val << ZPointerLoadShift) | store_good_mask
+      __ ld(tmp1, in_bytes(ZThreadLocalData::store_good_mask_offset()), R16_thread);
+      if (val != noreg) { // noreg encodes null.
+        __ rldimi(tmp1, val, ZPointerLoadShift, 0);
+      }
+    } else {
+      Label done;
+      Label medium;
+      Label medium_continuation; // bound in store_barrier_fast
+      Label slow;
+
+      store_barrier_fast(masm, base, ind_or_offs, val, tmp1, false, false, medium, medium_continuation);
+      __ b(done);
+      __ bind(medium);
+      store_barrier_medium(masm, base, ind_or_offs, tmp1, false, medium_continuation, slow);
+      __ bind(slow);
+      {
+        ZRuntimeCallSpill rcs(masm, noreg, preservation_level);
+        __ add(R3_ARG1, ind_or_offs, base);
+        __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), R3_ARG1);
+      }
+      __ b(medium_continuation);
+
+      __ bind(done);
+    }
+    BarrierSetAssembler::store_at(masm, decorators, type, base, ind_or_offs, tmp1, tmp2, tmp3, noreg, preservation_level);
+  } else {
+    BarrierSetAssembler::store_at(masm, decorators, type, base, ind_or_offs, val, tmp1, tmp2, tmp3, preservation_level);
+  }
 
   __ block_comment("} store_at (zgc)");
 }
-#endif // ASSERT
 
-void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType component_type,
+/* arraycopy */
+const Register _load_bad_mask = R6, _store_bad_mask = R7, _store_good_mask = R8;
+
+void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type,
                                               Register src, Register dst, Register count,
                                               Register preserve1, Register preserve2) {
-  __ block_comment("arraycopy_prologue (zgc) {");
+  bool is_checkcast_copy  = (decorators & ARRAYCOPY_CHECKCAST)   != 0,
+       dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 
-  /* ==== Check whether a special gc barrier is required for this particular load ==== */
-  if (!is_reference_type(component_type)) {
+  if (!ZBarrierSet::barrier_needed(decorators, type) || is_checkcast_copy) {
+    // Barrier not needed
     return;
   }
 
-  Label skip_barrier;
+  __ block_comment("arraycopy_prologue (zgc) {");
 
-  // Fast path: Array is of length zero
-  __ cmpdi(CCR0, count, 0);
-  __ beq(CCR0, skip_barrier);
-
-  /* ==== Ensure register sanity ==== */
-  Register tmp_R11 = R11_scratch1;
-
-  assert_different_registers(src, dst, count, tmp_R11, noreg);
-  if (preserve1 != noreg) {
-    // Not technically required, but unlikely being intended.
-    assert_different_registers(preserve1, preserve2);
-  }
-
-  /* ==== Invoke barrier (slowpath) ==== */
-  int nbytes_save = 0;
-
-  {
-    assert(!noreg->is_volatile(), "sanity");
-
-    if (preserve1->is_volatile()) {
-      __ std(preserve1, -BytesPerWord * ++nbytes_save, R1_SP);
-    }
-
-    if (preserve2->is_volatile() && preserve1 != preserve2) {
-      __ std(preserve2, -BytesPerWord * ++nbytes_save, R1_SP);
-    }
-
-    __ std(src, -BytesPerWord * ++nbytes_save, R1_SP);
-    __ std(dst, -BytesPerWord * ++nbytes_save, R1_SP);
-    __ std(count, -BytesPerWord * ++nbytes_save, R1_SP);
-
-    __ save_LR_CR(tmp_R11);
-    __ push_frame_reg_args(nbytes_save, tmp_R11);
-  }
-
-  // ZBarrierSetRuntime::load_barrier_on_oop_array_addr(src, count)
-  if (count == R3_ARG1) {
-    if (src == R4_ARG2) {
-      // Arguments are provided in reverse order
-      __ mr(tmp_R11, count);
-      __ mr(R3_ARG1, src);
-      __ mr(R4_ARG2, tmp_R11);
-    } else {
-      __ mr(R4_ARG2, count);
-      __ mr(R3_ARG1, src);
-    }
-  } else {
-    __ mr_if_needed(R3_ARG1, src);
-    __ mr_if_needed(R4_ARG2, count);
-  }
-
-  __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_array_addr());
-
-  __ pop_frame();
-  __ restore_LR_CR(tmp_R11);
-
-  {
-    __ ld(count, -BytesPerWord * nbytes_save--, R1_SP);
-    __ ld(dst, -BytesPerWord * nbytes_save--, R1_SP);
-    __ ld(src, -BytesPerWord * nbytes_save--, R1_SP);
-
-    if (preserve2->is_volatile() && preserve1 != preserve2) {
-      __ ld(preserve2, -BytesPerWord * nbytes_save--, R1_SP);
-    }
-
-    if (preserve1->is_volatile()) {
-      __ ld(preserve1, -BytesPerWord * nbytes_save--, R1_SP);
-    }
-  }
-
-  __ bind(skip_barrier);
+  load_copy_masks(masm, _load_bad_mask, _store_bad_mask, _store_good_mask, dest_uninitialized);
 
   __ block_comment("} arraycopy_prologue (zgc)");
 }
 
+void ZBarrierSetAssembler::load_copy_masks(MacroAssembler* masm,
+                                           Register load_bad_mask,
+                                           Register store_bad_mask,
+                                           Register store_good_mask,
+                                           bool dest_uninitialized) const {
+  __ ld(load_bad_mask, in_bytes(ZThreadLocalData::load_bad_mask_offset()), R16_thread);
+  __ ld(store_good_mask, in_bytes(ZThreadLocalData::store_good_mask_offset()), R16_thread);
+  if (dest_uninitialized) {
+    DEBUG_ONLY( __ li(store_bad_mask, -1); )
+  } else {
+    __ ld(store_bad_mask, in_bytes(ZThreadLocalData::store_bad_mask_offset()), R16_thread);
+  }
+}
+void ZBarrierSetAssembler::copy_load_at_fast(MacroAssembler* masm,
+                                             Register zpointer,
+                                             Register addr,
+                                             Register load_bad_mask,
+                                             Label& slow_path,
+                                             Label& continuation) const {
+  __ ldx(zpointer, addr);
+  __ and_(R0, zpointer, load_bad_mask);
+  __ bne(CCR0, slow_path);
+  __ bind(continuation);
+}
+void ZBarrierSetAssembler::copy_load_at_slow(MacroAssembler* masm,
+                                             Register zpointer,
+                                             Register addr,
+                                             Register tmp,
+                                             Label& slow_path,
+                                             Label& continuation) const {
+  __ align(32);
+  __ bind(slow_path);
+  __ mfctr(tmp); // preserve loop counter
+  {
+    ZRuntimeCallSpill rcs(masm, R0, MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS);
+    assert(zpointer != R4_ARG2, "or change argument setup");
+    __ mr_if_needed(R4_ARG2, addr);
+    __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(), zpointer, R4_ARG2);
+  }
+  __ sldi(zpointer, R0, ZPointerLoadShift); // Slow-path has uncolored; revert
+  __ mtctr(tmp); // restore loop counter
+  __ b(continuation);
+}
+void ZBarrierSetAssembler::copy_store_at_fast(MacroAssembler* masm,
+                                              Register zpointer,
+                                              Register addr,
+                                              Register store_bad_mask,
+                                              Register store_good_mask,
+                                              Label& medium_path,
+                                              Label& continuation,
+                                              bool dest_uninitialized) const {
+  if (!dest_uninitialized) {
+    __ ldx(R0, addr);
+    __ and_(R0, R0, store_bad_mask);
+    __ bne(CCR0, medium_path);
+    __ bind(continuation);
+  }
+  __ rldimi(zpointer, store_good_mask, 0, 64 - ZPointerLoadShift); // Replace color bits.
+  __ stdx(zpointer, addr);
+}
+void ZBarrierSetAssembler::copy_store_at_slow(MacroAssembler* masm,
+                                              Register addr,
+                                              Register tmp,
+                                              Label& medium_path,
+                                              Label& continuation,
+                                              bool dest_uninitialized) const {
+  if (!dest_uninitialized) {
+    Label slow_path;
+    __ align(32);
+    __ bind(medium_path);
+    store_barrier_medium(masm, addr, (intptr_t)0, tmp, false, continuation, slow_path);
+    __ bind(slow_path);
+    __ mfctr(tmp); // preserve loop counter
+    {
+      ZRuntimeCallSpill rcs(masm, noreg, MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS);
+      __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), addr);
+    }
+    __ mtctr(tmp); // restore loop counter
+    __ b(continuation);
+  }
+}
+
+// Arguments for generated stub:
+//      from:  R3_ARG1
+//      to:    R4_ARG2
+//      count: R5_ARG3 (int >= 0)
+void ZBarrierSetAssembler::generate_disjoint_oop_copy(MacroAssembler* masm, bool dest_uninitialized) {
+  const Register zpointer = R2, tmp = R9;
+  Label done, loop, load_bad, load_good, store_bad, store_good;
+  __ cmpdi(CCR0, R5_ARG3, 0);
+  __ beq(CCR0, done);
+  __ mtctr(R5_ARG3);
+
+  __ align(32);
+  __ bind(loop);
+  copy_load_at_fast(masm, zpointer, R3_ARG1, _load_bad_mask, load_bad, load_good);
+  copy_store_at_fast(masm, zpointer, R4_ARG2, _store_bad_mask, _store_good_mask, store_bad, store_good, dest_uninitialized);
+  __ addi(R3_ARG1, R3_ARG1, 8);
+  __ addi(R4_ARG2, R4_ARG2, 8);
+  __ bdnz(loop);
+
+  __ bind(done);
+  __ li(R3_RET, 0);
+  __ blr();
+
+  copy_load_at_slow(masm, zpointer, R3_ARG1, tmp, load_bad, load_good);
+  copy_store_at_slow(masm, R4_ARG2, tmp, store_bad, store_good, dest_uninitialized);
+}
+
+void ZBarrierSetAssembler::generate_conjoint_oop_copy(MacroAssembler* masm, bool dest_uninitialized) {
+  const Register zpointer = R2, tmp = R9;
+  Label done, loop, load_bad, load_good, store_bad, store_good;
+  __ sldi_(R0, R5_ARG3, 3);
+  __ beq(CCR0, done);
+  __ mtctr(R5_ARG3);
+  // Point behind last elements and copy backwards.
+  __ add(R3_ARG1, R3_ARG1, R0);
+  __ add(R4_ARG2, R4_ARG2, R0);
+
+  __ align(32);
+  __ bind(loop);
+  __ addi(R3_ARG1, R3_ARG1, -8);
+  __ addi(R4_ARG2, R4_ARG2, -8);
+  copy_load_at_fast(masm, zpointer, R3_ARG1, _load_bad_mask, load_bad, load_good);
+  copy_store_at_fast(masm, zpointer, R4_ARG2, _store_bad_mask, _store_good_mask, store_bad, store_good, dest_uninitialized);
+  __ bdnz(loop);
+
+  __ bind(done);
+  __ li(R3_RET, 0);
+  __ blr();
+
+  copy_load_at_slow(masm, zpointer, R3_ARG1, tmp, load_bad, load_good);
+  copy_store_at_slow(masm, R4_ARG2, tmp, store_bad, store_good, dest_uninitialized);
+}
+
+
+// Verify a colored pointer.
+void ZBarrierSetAssembler::check_oop(MacroAssembler *masm, Register obj, const char* msg) {
+  if (!VerifyOops) {
+    return;
+  }
+  Label done, skip_uncolor;
+  // Skip (colored) null.
+  __ srdi_(R0, obj, ZPointerLoadShift);
+  __ beq(CCR0, done);
+
+  // Check if ZAddressHeapBase << ZPointerLoadShift is set. If so, we need to uncolor.
+  __ rldicl_(R0, obj, 64 - ZAddressHeapBaseShift - ZPointerLoadShift, 63);
+  __ mr(R0, obj);
+  __ beq(CCR0, skip_uncolor);
+  __ srdi(R0, obj, ZPointerLoadShift);
+  __ bind(skip_uncolor);
+
+  __ verify_oop(R0, msg);
+  __ bind(done);
+}
+
+
 void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
                                                          Register obj, Register tmp, Label& slowpath) {
   __ block_comment("try_resolve_jobject_in_native (zgc) {");
 
-  assert_different_registers(jni_env, obj, tmp);
+  Label done, tagged, weak_tagged, check_color;
+  Address load_bad_mask = load_bad_mask_from_jni_env(jni_env),
+          mark_bad_mask = mark_bad_mask_from_jni_env(jni_env);
 
-  // Resolve the pointer using the standard implementation for weak tag handling and pointer verification.
-  BarrierSetAssembler::try_resolve_jobject_in_native(masm, dst, jni_env, obj, tmp, slowpath);
+  // Test for tag
+  __ andi_(tmp, obj, JNIHandles::tag_mask);
+  __ bne(CCR0, tagged);
 
-  // Check whether pointer is dirty.
-  __ ld(tmp,
-        in_bytes(ZThreadLocalData::address_bad_mask_offset() - JavaThread::jni_environment_offset()),
-        jni_env);
+  // Resolve local handle
+  __ ld(dst, 0, obj);
+  __ b(done);
 
-  __ and_(tmp, obj, tmp);
+  __ bind(tagged);
+
+  // Test for weak tag
+  __ andi_(tmp, obj, JNIHandles::TypeTag::weak_global);
+  __ clrrdi(dst, obj, JNIHandles::tag_size); // Untag.
+  __ bne(CCR0, weak_tagged);
+
+  // Resolve global handle
+  __ ld(dst, 0, dst);
+  __ ld(tmp, load_bad_mask.disp(), load_bad_mask.base());
+  __ b(check_color);
+
+  __ bind(weak_tagged);
+
+  // Resolve weak handle
+  __ ld(dst, 0, dst);
+  __ ld(tmp, mark_bad_mask.disp(), mark_bad_mask.base());
+
+  __ bind(check_color);
+  __ and_(tmp, tmp, dst);
   __ bne(CCR0, slowpath);
 
+  // Uncolor
+  __ srdi(dst, dst, ZPointerLoadShift);
+
+  __ bind(done);
+
   __ block_comment("} try_resolve_jobject_in_native (zgc)");
 }
 
@@ -289,17 +635,40 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm, R
 #ifdef COMPILER1
 #define __ ce->masm()->
 
-// Code emitted by LIR node "LIR_OpZLoadBarrierTest" which in turn is emitted by ZBarrierSetC1::load_barrier.
-// The actual compare and branch instructions are represented as stand-alone LIR nodes.
-void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
-                                                         LIR_Opr ref) const {
-  __ block_comment("load_barrier_test (zgc) {");
+static void z_uncolor(LIR_Assembler* ce, LIR_Opr ref) {
+  Register r = ref->as_register();
+  __ srdi(r, r, ZPointerLoadShift);
+}
 
-  __ ld(R0, in_bytes(ZThreadLocalData::address_bad_mask_offset()), R16_thread);
-  __ andr(R0, R0, ref->as_pointer_register());
-  __ cmpdi(CCR5 /* as mandated by LIR node */, R0, 0);
+static void check_color(LIR_Assembler* ce, LIR_Opr ref, bool on_non_strong) {
+  int relocFormat = on_non_strong ? ZBarrierRelocationFormatMarkBadMask
+                                  : ZBarrierRelocationFormatLoadBadMask;
+  __ relocate(barrier_Relocation::spec(), relocFormat);
+  __ andi_(R0, ref->as_register(), barrier_Relocation::unpatched);
+}
 
-  __ block_comment("} load_barrier_test (zgc)");
+static void z_color(LIR_Assembler* ce, LIR_Opr ref) {
+  __ sldi(ref->as_register(), ref->as_register(), ZPointerLoadShift);
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBits);
+  __ ori(ref->as_register(), ref->as_register(), barrier_Relocation::unpatched);
+}
+
+void ZBarrierSetAssembler::generate_c1_uncolor(LIR_Assembler* ce, LIR_Opr ref) const {
+  z_uncolor(ce, ref);
+}
+
+void ZBarrierSetAssembler::generate_c1_color(LIR_Assembler* ce, LIR_Opr ref) const {
+  z_color(ce, ref);
+}
+
+void ZBarrierSetAssembler::generate_c1_load_barrier(LIR_Assembler* ce,
+                                                    LIR_Opr ref,
+                                                    ZLoadBarrierStubC1* stub,
+                                                    bool on_non_strong) const {
+  check_color(ce, ref, on_non_strong);
+  __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CCR0, Assembler::equal), *stub->entry());
+  z_uncolor(ce, ref);
+  __ bind(*stub->continuation());
 }
 
 // Code emitted by code stub "ZLoadBarrierStubC1" which in turn is emitted by ZBarrierSetC1::load_barrier.
@@ -332,19 +701,77 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
 
   /* ==== Invoke stub ==== */
   // Pass arguments via stack. The stack pointer will be bumped by the stub.
-  __ std(ref, (intptr_t) -1 * BytesPerWord, R1_SP);
-  __ std(ref_addr, (intptr_t) -2 * BytesPerWord, R1_SP);
+  __ std(ref, -1 * BytesPerWord, R1_SP);
+  __ std(ref_addr, -2 * BytesPerWord, R1_SP);
 
-  __ load_const_optimized(R0, stub->runtime_stub());
+  __ load_const_optimized(R0, stub->runtime_stub(), /* temp */ ref);
   __ call_stub(R0);
 
   // The runtime stub passes the result via the R0 register, overriding the previously-loaded stub address.
-  __ mr_if_needed(ref, R0);
+  __ mr(ref, R0);
   __ b(*stub->continuation());
 
   __ block_comment("} c1_load_barrier_stub (zgc)");
 }
 
+void ZBarrierSetAssembler::generate_c1_store_barrier(LIR_Assembler* ce,
+                                                     LIR_Address* addr,
+                                                     LIR_Opr new_zaddress,
+                                                     LIR_Opr new_zpointer,
+                                                     ZStoreBarrierStubC1* stub) const {
+  Register rnew_zaddress = new_zaddress->as_register();
+  Register rnew_zpointer = new_zpointer->as_register();
+
+  Register rbase = addr->base()->as_pointer_register();
+  RegisterOrConstant ind_or_offs = (addr->index()->is_illegal())
+                                 ? (RegisterOrConstant)addr->disp()
+                                 : (RegisterOrConstant)addr->index()->as_pointer_register();
+
+  store_barrier_fast(ce->masm(),
+                     rbase,
+                     ind_or_offs,
+                     rnew_zaddress,
+                     rnew_zpointer,
+                     true,
+                     stub->is_atomic(),
+                     *stub->entry(),
+                     *stub->continuation());
+}
+
+void ZBarrierSetAssembler::generate_c1_store_barrier_stub(LIR_Assembler* ce,
+                                                          ZStoreBarrierStubC1* stub) const {
+  // Stub entry
+  __ bind(*stub->entry());
+
+  Label slow;
+
+  LIR_Address* addr = stub->ref_addr()->as_address_ptr();
+  assert(addr->index()->is_illegal() || addr->disp() == 0, "can't have both");
+  Register rbase = addr->base()->as_pointer_register();
+  RegisterOrConstant ind_or_offs = (addr->index()->is_illegal())
+                                 ? (RegisterOrConstant)addr->disp()
+                                 : (RegisterOrConstant)addr->index()->as_pointer_register();
+  Register new_zpointer = stub->new_zpointer()->as_register();
+
+  store_barrier_medium(ce->masm(),
+                       rbase,
+                       ind_or_offs,
+                       new_zpointer, // temp
+                       stub->is_atomic(),
+                       *stub->continuation(),
+                       slow);
+
+  __ bind(slow);
+
+  __ load_const_optimized(/*stub address*/ new_zpointer, stub->runtime_stub(), R0);
+  __ add(R0, ind_or_offs, rbase); // pass store address in R0
+  __ mtctr(new_zpointer);
+  __ bctrl();
+
+  // Stub exit
+  __ b(*stub->continuation());
+}
+
 #undef __
 #define __ sasm->
 
@@ -360,8 +787,8 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler*
   __ save_LR_CR(R0);
 
   // Load arguments back again from the stack.
-  __ ld(R3_ARG1, (intptr_t) -1 * BytesPerWord, R1_SP); // ref
-  __ ld(R4_ARG2, (intptr_t) -2 * BytesPerWord, R1_SP); // ref_addr
+  __ ld(R3_ARG1, -1 * BytesPerWord, R1_SP); // ref
+  __ ld(R4_ARG2, -2 * BytesPerWord, R1_SP); // ref_addr
 
   __ push_frame_reg_args(nbytes_save, R0);
 
@@ -379,6 +806,32 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler*
   __ block_comment("} c1_load_barrier_runtime_stub (zgc)");
 }
 
+void ZBarrierSetAssembler::generate_c1_store_barrier_runtime_stub(StubAssembler* sasm,
+                                                                  bool self_healing) const {
+  __ block_comment("c1_store_barrier_runtime_stub (zgc) {");
+
+  const int nbytes_save = MacroAssembler::num_volatile_regs * BytesPerWord;
+  __ save_volatile_gprs(R1_SP, -nbytes_save);
+  __ mr(R3_ARG1, R0); // store address
+
+  __ save_LR_CR(R0);
+  __ push_frame_reg_args(nbytes_save, R0);
+
+  if (self_healing) {
+    __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr());
+  } else {
+    __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr());
+  }
+
+  __ pop_frame();
+  __ restore_LR_CR(R3_RET);
+  __ restore_volatile_gprs(R1_SP, -nbytes_save);
+
+  __ blr();
+
+  __ block_comment("} c1_store_barrier_runtime_stub (zgc)");
+}
+
 #undef __
 #endif // COMPILER1
 
@@ -406,8 +859,8 @@ class ZSaveLiveRegisters {
   int _frame_size;
 
  public:
-  ZSaveLiveRegisters(MacroAssembler *masm, ZLoadBarrierStubC2 *stub)
-      : _masm(masm), _reg_mask(stub->live()), _result_reg(stub->ref()) {
+  ZSaveLiveRegisters(MacroAssembler *masm, ZBarrierStubC2 *stub)
+      : _masm(masm), _reg_mask(stub->live()), _result_reg(stub->result()) {
 
     const int register_save_size = iterate_over_register_mask(ACTION_COUNT_ONLY) * BytesPerWord;
     _frame_size = align_up(register_save_size, frame::alignment_in_bytes)
@@ -559,6 +1012,7 @@ class ZSetupArguments {
 #define __ masm->
 
 void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const {
+  Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
   __ block_comment("generate_c2_load_barrier_stub (zgc) {");
 
   __ bind(*stub->entry());
@@ -581,5 +1035,79 @@ void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, Z
   __ block_comment("} generate_c2_load_barrier_stub (zgc)");
 }
 
+void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm, ZStoreBarrierStubC2* stub) const {
+  Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
+  __ block_comment("ZStoreBarrierStubC2");
+
+  // Stub entry
+  __ bind(*stub->entry());
+
+  Label slow;
+
+  Address addr = stub->ref_addr();
+  Register rbase = addr.base();
+  RegisterOrConstant ind_or_offs = (addr.index() == noreg)
+                                 ? (RegisterOrConstant)addr.disp()
+                                 : (RegisterOrConstant)addr.index();
+
+  if (!stub->is_native()) {
+    store_barrier_medium(masm,
+                         rbase,
+                         ind_or_offs,
+                         stub->new_zpointer(),
+                         stub->is_atomic(),
+                         *stub->continuation(),
+                         slow);
+  }
+
+  __ bind(slow);
+  {
+    ZSaveLiveRegisters save_live_registers(masm, stub);
+    __ add(R3_ARG1, ind_or_offs, rbase);
+    if (stub->is_native()) {
+      __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr(), R3_ARG1);
+    } else if (stub->is_atomic()) {
+      __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr(), R3_ARG1);
+    } else {
+      __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), R3_ARG1);
+    }
+  }
+
+  // Stub exit
+  __ b(*stub->continuation());
+}
+
 #undef __
 #endif // COMPILER2
+
+static uint16_t patch_barrier_relocation_value(int format) {
+  switch (format) {
+  case ZBarrierRelocationFormatLoadBadMask:
+    return (uint16_t)ZPointerLoadBadMask;
+  case ZBarrierRelocationFormatMarkBadMask:
+    return (uint16_t)ZPointerMarkBadMask;
+  case ZBarrierRelocationFormatStoreGoodBits:
+    return (uint16_t)ZPointerStoreGoodMask;
+  case ZBarrierRelocationFormatStoreBadMask:
+    return (uint16_t)ZPointerStoreBadMask;
+  default:
+    ShouldNotReachHere();
+    return 0;
+  }
+}
+
+void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
+#ifdef ASSERT
+  int inst = *(int*)addr;
+  if (format == ZBarrierRelocationFormatStoreGoodBits) {
+    assert(Assembler::is_li(inst) || Assembler::is_ori(inst) || Assembler::is_cmpli(inst),
+           "unexpected instruction 0x%04x", inst);
+    // Note: li uses sign extend, but these bits will get cleared by rldimi.
+  } else {
+    assert(Assembler::is_andi(inst), "unexpected instruction 0x%04x", inst);
+  }
+#endif
+  // Patch the signed/unsigned 16 bit immediate field of the instruction.
+  *(uint16_t*)(addr BIG_ENDIAN_ONLY(+2)) = patch_barrier_relocation_value(format);
+  ICache::ppc64_flush_icache_bytes(addr, BytesPerInstWord);
+}
diff --git a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp
index 808edca6939..7aca78db7d0 100644
--- a/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp
+++ b/src/hotspot/cpu/ppc/gc/z/zBarrierSetAssembler_ppc.hpp
@@ -32,17 +32,27 @@
 #endif // COMPILER2
 
 #ifdef COMPILER1
+class CodeStub;
+class LIR_Address;
 class LIR_Assembler;
 class LIR_Opr;
 class StubAssembler;
 class ZLoadBarrierStubC1;
+class ZStoreBarrierStubC1;
 #endif // COMPILER1
 
 #ifdef COMPILER2
+class MachNode;
 class Node;
 class ZLoadBarrierStubC2;
+class ZStoreBarrierStubC2;
 #endif // COMPILER2
 
+const int ZBarrierRelocationFormatLoadBadMask = 0;
+const int ZBarrierRelocationFormatMarkBadMask = 1;
+const int ZBarrierRelocationFormatStoreGoodBits = 2;
+const int ZBarrierRelocationFormatStoreBadMask = 3;
+
 class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase {
 public:
   virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
@@ -50,12 +60,10 @@ public:
                        Register tmp1, Register tmp2,
                        MacroAssembler::PreservationLevel preservation_level, Label *L_handle_null = nullptr);
 
-#ifdef ASSERT
   virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                         Register base, RegisterOrConstant ind_or_offs, Register val,
                         Register tmp1, Register tmp2, Register tmp3,
                         MacroAssembler::PreservationLevel preservation_level);
-#endif // ASSERT
 
   virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
                                   Register src, Register dst, Register count,
@@ -64,24 +72,102 @@ public:
   virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register dst, Register jni_env,
                                              Register obj, Register tmp, Label& slowpath);
 
-  virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
+  virtual void check_oop(MacroAssembler *masm, Register obj, const char* msg);
+
+  virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_instruction_and_data_patch; }
 
 #ifdef COMPILER1
-  void generate_c1_load_barrier_test(LIR_Assembler* ce,
-                                     LIR_Opr ref) const;
-
   void generate_c1_load_barrier_stub(LIR_Assembler* ce,
                                      ZLoadBarrierStubC1* stub) const;
 
   void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
                                              DecoratorSet decorators) const;
+
+  void generate_c1_color(LIR_Assembler* ce, LIR_Opr ref) const;
+  void generate_c1_uncolor(LIR_Assembler* ce, LIR_Opr ref) const;
+
+  void generate_c1_load_barrier(LIR_Assembler* ce,
+                                LIR_Opr ref,
+                                ZLoadBarrierStubC1* stub,
+                                bool on_non_strong) const;
+
+  void generate_c1_store_barrier(LIR_Assembler* ce,
+                                 LIR_Address* addr,
+                                 LIR_Opr new_zaddress,
+                                 LIR_Opr new_zpointer,
+                                 ZStoreBarrierStubC1* stub) const;
+
+  void generate_c1_store_barrier_stub(LIR_Assembler* ce,
+                                      ZStoreBarrierStubC1* stub) const;
+
+  void generate_c1_store_barrier_runtime_stub(StubAssembler* sasm,
+                                              bool self_healing) const;
 #endif // COMPILER1
 
 #ifdef COMPILER2
   OptoReg::Name refine_register(const Node* node, OptoReg::Name opto_reg) const;
 
   void generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const;
+
+  void generate_c2_store_barrier_stub(MacroAssembler* masm, ZStoreBarrierStubC2* stub) const;
 #endif // COMPILER2
+
+  void store_barrier_fast(MacroAssembler* masm,
+                          Register ref_base,
+                          RegisterOrConstant ind_or_offset,
+                          Register rnew_persistent,
+                          Register rnew_transient,
+                          bool in_nmethod,
+                          bool is_atomic,
+                          Label& medium_path,
+                          Label& medium_path_continuation) const;
+
+  void store_barrier_medium(MacroAssembler* masm,
+                            Register ref_base,
+                            RegisterOrConstant ind_or_offs,
+                            Register tmp,
+                            bool is_atomic,
+                            Label& medium_path_continuation,
+                            Label& slow_path) const;
+
+  void load_copy_masks(MacroAssembler* masm,
+                       Register load_bad_mask,
+                       Register store_bad_mask,
+                       Register store_good_mask,
+                       bool dest_uninitialized) const;
+  void copy_load_at_fast(MacroAssembler* masm,
+                         Register zpointer,
+                         Register addr,
+                         Register load_bad_mask,
+                         Label& slow_path,
+                         Label& continuation) const;
+  void copy_load_at_slow(MacroAssembler* masm,
+                         Register zpointer,
+                         Register addr,
+                         Register tmp,
+                         Label& slow_path,
+                         Label& continuation) const;
+  void copy_store_at_fast(MacroAssembler* masm,
+                          Register zpointer,
+                          Register addr,
+                          Register store_bad_mask,
+                          Register store_good_mask,
+                          Label& medium_path,
+                          Label& continuation,
+                          bool dest_uninitialized) const;
+  void copy_store_at_slow(MacroAssembler* masm,
+                          Register addr,
+                          Register tmp,
+                          Label& medium_path,
+                          Label& continuation,
+                          bool dest_uninitialized) const;
+
+  void generate_disjoint_oop_copy(MacroAssembler* masm, bool dest_uninitialized);
+  void generate_conjoint_oop_copy(MacroAssembler* masm, bool dest_uninitialized);
+
+  void patch_barrier_relocation(address addr, int format);
+
+  void patch_barriers() {}
 };
 
-#endif // CPU_AARCH64_GC_Z_ZBARRIERSETASSEMBLER_AARCH64_HPP
+#endif // CPU_PPC_GC_Z_ZBARRIERSETASSEMBLER_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/gc/z/zGlobals_ppc.hpp b/src/hotspot/cpu/ppc/gc/z/zGlobals_ppc.hpp
index 85d112b437a..81c72747ac8 100644
--- a/src/hotspot/cpu/ppc/gc/z/zGlobals_ppc.hpp
+++ b/src/hotspot/cpu/ppc/gc/z/zGlobals_ppc.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2021 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -26,10 +26,7 @@
 #define CPU_PPC_GC_Z_ZGLOBALS_PPC_HPP
 
 #include "globalDefinitions_ppc.hpp"
-const size_t ZPlatformHeapViews        = 3;
+
 const size_t ZPlatformCacheLineSize    = DEFAULT_CACHE_LINE_SIZE;
 
-size_t ZPlatformAddressOffsetBits();
-size_t ZPlatformAddressMetadataShift();
-
 #endif // CPU_PPC_GC_Z_ZGLOBALS_PPC_HPP
diff --git a/src/hotspot/cpu/ppc/gc/z/z_ppc.ad b/src/hotspot/cpu/ppc/gc/z/z_ppc.ad
index a8ce64ed1d9..777e5a785a7 100644
--- a/src/hotspot/cpu/ppc/gc/z/z_ppc.ad
+++ b/src/hotspot/cpu/ppc/gc/z/z_ppc.ad
@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
 // Copyright (c) 2021 SAP SE. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
@@ -32,51 +32,73 @@ source_hpp %{
 
 source %{
 
-static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref,
-                           Register tmp, uint8_t barrier_data) {
-  if (barrier_data == ZLoadBarrierElided) {
-    return;
-  }
+#include "gc/z/zBarrierSetAssembler.hpp"
 
-  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
-  __ ld(tmp, in_bytes(ZThreadLocalData::address_bad_mask_offset()), R16_thread);
-  __ and_(tmp, tmp, ref);
-  __ bne_far(CCR0, *stub->entry(), MacroAssembler::bc_far_optimize_on_relocate);
-  __ bind(*stub->continuation());
+static void z_color(MacroAssembler& _masm, Register dst, Register src) {
+  assert_different_registers(dst, src);
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodBits);
+  __ li(dst, barrier_Relocation::unpatched); // Load color bits.
+  if (src == noreg) { // noreg encodes null.
+    if (ZPointerLoadShift >= 16) {
+      __ rldicl(dst, dst, 0, 64 - ZPointerLoadShift); // Clear sign extension from li.
+    }
+  } else {
+    __ rldimi(dst, src, ZPointerLoadShift, 0); // Insert shifted pointer.
+  }
 }
 
-static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref,
-                                     Register tmp) {
-  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, ZLoadBarrierStrong);
-  __ b(*stub->entry());
-  __ bind(*stub->continuation());
+static void z_uncolor(MacroAssembler& _masm, Register ref) {
+  __ srdi(ref, ref, ZPointerLoadShift);
+}
+
+static void check_color(MacroAssembler& _masm, Register ref, bool on_non_strong) {
+  int relocFormat = on_non_strong ? ZBarrierRelocationFormatMarkBadMask
+                                  : ZBarrierRelocationFormatLoadBadMask;
+  __ relocate(barrier_Relocation::spec(), relocFormat);
+  __ andi_(R0, ref, barrier_Relocation::unpatched);
+}
+
+static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref) {
+  Assembler::InlineSkippedInstructionsCounter skipped_counter(&_masm);
+  if (node->barrier_data() == ZBarrierElided) {
+    z_uncolor(_masm, ref);
+  } else {
+    const bool on_non_strong =
+      ((node->barrier_data() & ZBarrierWeak) != 0) ||
+      ((node->barrier_data() & ZBarrierPhantom) != 0);
+
+    check_color(_masm, ref, on_non_strong);
+
+    ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref);
+    __ bne_far(CCR0, *stub->entry(), MacroAssembler::bc_far_optimize_on_relocate);
+
+    z_uncolor(_masm, ref);
+    __ bind(*stub->continuation());
+  }
+}
+
+static void z_store_barrier(MacroAssembler& _masm, const MachNode* node, Register ref_base, intptr_t disp, Register rnew_zaddress, Register rnew_zpointer, bool is_atomic) {
+  Assembler::InlineSkippedInstructionsCounter skipped_counter(&_masm);
+  if (node->barrier_data() == ZBarrierElided) {
+    z_color(_masm, rnew_zpointer, rnew_zaddress);
+  } else {
+    bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
+    ZStoreBarrierStubC2* const stub = ZStoreBarrierStubC2::create(node, Address(ref_base, disp), rnew_zaddress, rnew_zpointer, is_native, is_atomic);
+    ZBarrierSetAssembler* bs_asm = ZBarrierSet::assembler();
+    bs_asm->store_barrier_fast(&_masm, ref_base, disp, rnew_zaddress, rnew_zpointer, true /* in_nmethod */, is_atomic, *stub->entry(), *stub->continuation());
+  }
 }
 
 static void z_compare_and_swap(MacroAssembler& _masm, const MachNode* node,
                               Register res, Register mem, Register oldval, Register newval,
-                              Register tmp_xchg, Register tmp_mask,
-                              bool weak, bool acquire) {
-  // z-specific load barrier requires strong CAS operations.
-  // Weak CAS operations are thus only emitted if the barrier is elided.
-  __ cmpxchgd(CCR0, tmp_xchg, oldval, newval, mem,
-              MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), res, NULL, true,
-              weak && node->barrier_data() == ZLoadBarrierElided);
+                              Register tmp1, Register tmp2, bool acquire) {
 
-  if (node->barrier_data() != ZLoadBarrierElided) {
-    Label skip_barrier;
-
-    __ ld(tmp_mask, in_bytes(ZThreadLocalData::address_bad_mask_offset()), R16_thread);
-    __ and_(tmp_mask, tmp_mask, tmp_xchg);
-    __ beq(CCR0, skip_barrier);
-
-    // CAS must have failed because pointer in memory is bad.
-    z_load_barrier_slow_path(_masm, node, Address(mem), tmp_xchg, res /* used as tmp */);
-
-    __ cmpxchgd(CCR0, tmp_xchg, oldval, newval, mem,
-                MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), res, NULL, true, weak);
-
-    __ bind(skip_barrier);
-  }
+  Register rold_zpointer = tmp1, rnew_zpointer = tmp2;
+  z_store_barrier(_masm, node, mem, 0, newval, rnew_zpointer, true /* is_atomic */);
+  z_color(_masm, rold_zpointer, oldval);
+  __ cmpxchgd(CCR0, R0, rold_zpointer, rnew_zpointer, mem,
+              MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), res, nullptr, true,
+              false /* we could support weak, but benefit is questionable */);
 
   if (acquire) {
     if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
@@ -90,27 +112,16 @@ static void z_compare_and_swap(MacroAssembler& _masm, const MachNode* node,
 }
 
 static void z_compare_and_exchange(MacroAssembler& _masm, const MachNode* node,
-                                   Register res, Register mem, Register oldval, Register newval, Register tmp,
-                                   bool weak, bool acquire) {
-  // z-specific load barrier requires strong CAS operations.
-  // Weak CAS operations are thus only emitted if the barrier is elided.
-  __ cmpxchgd(CCR0, res, oldval, newval, mem,
-              MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), noreg, NULL, true,
-              weak && node->barrier_data() == ZLoadBarrierElided);
+                                   Register res, Register mem, Register oldval, Register newval,
+                                   Register tmp, bool acquire) {
 
-  if (node->barrier_data() != ZLoadBarrierElided) {
-    Label skip_barrier;
-    __ ld(tmp, in_bytes(ZThreadLocalData::address_bad_mask_offset()), R16_thread);
-    __ and_(tmp, tmp, res);
-    __ beq(CCR0, skip_barrier);
-
-    z_load_barrier_slow_path(_masm, node, Address(mem), res, tmp);
-
-    __ cmpxchgd(CCR0, res, oldval, newval, mem,
-                MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), noreg, NULL, true, weak);
-
-    __ bind(skip_barrier);
-  }
+  Register rold_zpointer = R0, rnew_zpointer = tmp;
+  z_store_barrier(_masm, node, mem, 0, newval, rnew_zpointer, true /* is_atomic */);
+  z_color(_masm, rold_zpointer, oldval);
+  __ cmpxchgd(CCR0, res, rold_zpointer, rnew_zpointer, mem,
+              MacroAssembler::MemBarNone, MacroAssembler::cmpxchgx_hint_atomic_update(), noreg, nullptr, true,
+              false /* we could support weak, but benefit is questionable */);
+  z_uncolor(_masm, res);
 
   if (acquire) {
     if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
@@ -125,114 +136,110 @@ static void z_compare_and_exchange(MacroAssembler& _masm, const MachNode* node,
 
 %}
 
-instruct zLoadP(iRegPdst dst, memoryAlg4 mem, iRegPdst tmp, flagsRegCR0 cr0)
+instruct zLoadP(iRegPdst dst, memoryAlg4 mem, flagsRegCR0 cr0)
 %{
   match(Set dst (LoadP mem));
-  effect(TEMP_DEF dst, TEMP tmp, KILL cr0);
+  effect(TEMP_DEF dst, KILL cr0);
   ins_cost(MEMORY_REF_COST);
 
-  predicate((UseZGC && n->as_Load()->barrier_data() != 0)
+  predicate((UseZGC && ZGenerational && n->as_Load()->barrier_data() != 0)
             && (n->as_Load()->is_unordered() || followed_by_acquire(n)));
 
   format %{ "LD $dst, $mem" %}
   ins_encode %{
     assert($mem$$index == 0, "sanity");
     __ ld($dst$$Register, $mem$$disp, $mem$$base$$Register);
-    z_load_barrier(_masm, this, Address($mem$$base$$Register, $mem$$disp), $dst$$Register, $tmp$$Register, barrier_data());
+    z_load_barrier(_masm, this, Address($mem$$base$$Register, $mem$$disp), $dst$$Register);
   %}
   ins_pipe(pipe_class_default);
 %}
 
 // Load Pointer Volatile
-instruct zLoadP_acq(iRegPdst dst, memoryAlg4 mem, iRegPdst tmp, flagsRegCR0 cr0)
+instruct zLoadP_acq(iRegPdst dst, memoryAlg4 mem, flagsRegCR0 cr0)
 %{
   match(Set dst (LoadP mem));
-  effect(TEMP_DEF dst, TEMP tmp, KILL cr0);
+  effect(TEMP_DEF dst, KILL cr0);
   ins_cost(3 * MEMORY_REF_COST);
 
   // Predicate on instruction order is implicitly present due to the predicate of the cheaper zLoadP operation
-  predicate(UseZGC && n->as_Load()->barrier_data() != 0);
+  predicate(UseZGC && ZGenerational && n->as_Load()->barrier_data() != 0);
 
   format %{ "LD acq $dst, $mem" %}
   ins_encode %{
     __ ld($dst$$Register, $mem$$disp, $mem$$base$$Register);
-    z_load_barrier(_masm, this, Address($mem$$base$$Register, $mem$$disp), $dst$$Register, $tmp$$Register, barrier_data());
+    z_load_barrier(_masm, this, Address($mem$$base$$Register, $mem$$disp), $dst$$Register);
 
     // Uses the isync instruction as an acquire barrier.
     // This exploits the compare and the branch in the z load barrier (load, compare and branch, isync).
+    if (barrier_data() == ZBarrierElided) __ twi_0($dst$$Register);
     __ isync();
   %}
   ins_pipe(pipe_class_default);
 %}
 
-instruct zCompareAndSwapP(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
-                          iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
-  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
-  effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
+// Store Pointer
+instruct zStoreP(memoryAlg4 mem, iRegPsrc src, iRegPdst tmp, flagsRegCR0 cr0)
+%{
+  predicate(UseZGC && ZGenerational && n->as_Store()->barrier_data() != 0);
+  match(Set mem (StoreP mem src));
+  effect(TEMP tmp, KILL cr0);
+  ins_cost(2 * MEMORY_REF_COST);
+  format %{ "std    $mem, $src\t# ptr" %}
+  ins_encode %{
+    z_store_barrier(_masm, this, $mem$$base$$Register, $mem$$disp, $src$$Register, $tmp$$Register, false /* is_atomic */);
+    __ std($tmp$$Register, $mem$$disp, $mem$$base$$Register);
+  %}
+  ins_pipe(pipe_class_default);
+%}
 
-  predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)
+instruct zStorePNull(memoryAlg4 mem, immP_0 zero, iRegPdst tmp, flagsRegCR0 cr0)
+%{
+  predicate(UseZGC && ZGenerational && n->as_Store()->barrier_data() != 0);
+  match(Set mem (StoreP mem zero));
+  effect(TEMP tmp, KILL cr0);
+  ins_cost(MEMORY_REF_COST);
+  format %{ "std    $mem, null\t# ptr" %}
+  ins_encode %{
+    z_store_barrier(_masm, this, $mem$$base$$Register, $mem$$disp, noreg, $tmp$$Register, false /* is_atomic */);
+    __ std($tmp$$Register, $mem$$disp, $mem$$base$$Register);
+  %}
+  ins_pipe(pipe_class_default);
+%}
+
+instruct zCompareAndSwapP(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
+                          iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr0) %{
+  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+  effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr0);
+
+  predicate((UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0)
             && (((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*) n)->order() != MemNode::seqcst));
 
   format %{ "CMPXCHG $res, $mem, $oldval, $newval; as bool; ptr" %}
   ins_encode %{
     z_compare_and_swap(_masm, this,
                        $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
-                       $tmp_xchg$$Register, $tmp_mask$$Register,
-                       false /* weak */, false /* acquire */);
+                       $tmp1$$Register, $tmp2$$Register,
+                       false /* acquire */);
   %}
   ins_pipe(pipe_class_default);
 %}
 
 instruct zCompareAndSwapP_acq(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
-                              iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
+                              iRegPdst tmp1, iRegPdst tmp2, flagsRegCR0 cr0) %{
   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
-  effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
+  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+  effect(TEMP_DEF res, TEMP tmp1, TEMP tmp2, KILL cr0);
 
-  predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)
+  predicate((UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0)
             && (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*) n)->order() == MemNode::seqcst));
 
   format %{ "CMPXCHG acq $res, $mem, $oldval, $newval; as bool; ptr" %}
   ins_encode %{
     z_compare_and_swap(_masm, this,
                        $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
-                       $tmp_xchg$$Register, $tmp_mask$$Register,
-                       false /* weak */, true /* acquire */);
-  %}
-  ins_pipe(pipe_class_default);
-%}
-
-instruct zCompareAndSwapPWeak(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
-                              iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
-  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
-  effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
-
-  predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)
-            && ((CompareAndSwapNode*)n)->order() != MemNode::acquire && ((CompareAndSwapNode*) n)->order() != MemNode::seqcst);
-
-  format %{ "weak CMPXCHG $res, $mem, $oldval, $newval; as bool; ptr" %}
-  ins_encode %{
-    z_compare_and_swap(_masm, this,
-                       $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
-                       $tmp_xchg$$Register, $tmp_mask$$Register,
-                       true /* weak */, false /* acquire */);
-  %}
-  ins_pipe(pipe_class_default);
-%}
-
-instruct zCompareAndSwapPWeak_acq(iRegIdst res, iRegPdst mem, iRegPsrc oldval, iRegPsrc newval,
-                                  iRegPdst tmp_xchg, iRegPdst tmp_mask, flagsRegCR0 cr0) %{
-  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
-  effect(TEMP_DEF res, TEMP tmp_xchg, TEMP tmp_mask, KILL cr0);
-
-  predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)
-            && (((CompareAndSwapNode*)n)->order() == MemNode::acquire || ((CompareAndSwapNode*) n)->order() == MemNode::seqcst));
-
-  format %{ "weak CMPXCHG acq $res, $mem, $oldval, $newval; as bool; ptr" %}
-  ins_encode %{
-    z_compare_and_swap(_masm, this,
-                       $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register,
-                       $tmp_xchg$$Register, $tmp_mask$$Register,
-                       true /* weak */, true /* acquire */);
+                       $tmp1$$Register, $tmp2$$Register,
+                       true /* acquire */);
   %}
   ins_pipe(pipe_class_default);
 %}
@@ -242,7 +249,7 @@ instruct zCompareAndExchangeP(iRegPdst res, iRegPdst mem, iRegPsrc oldval, iRegP
   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
   effect(TEMP_DEF res, TEMP tmp, KILL cr0);
 
-  predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)
+  predicate((UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0)
             && (
               ((CompareAndSwapNode*)n)->order() != MemNode::acquire
               && ((CompareAndSwapNode*)n)->order() != MemNode::seqcst
@@ -252,7 +259,7 @@ instruct zCompareAndExchangeP(iRegPdst res, iRegPdst mem, iRegPsrc oldval, iRegP
   ins_encode %{
     z_compare_and_exchange(_masm, this,
                            $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, $tmp$$Register,
-                           false /* weak */, false /* acquire */);
+                           false /* acquire */);
   %}
   ins_pipe(pipe_class_default);
 %}
@@ -262,7 +269,7 @@ instruct zCompareAndExchangeP_acq(iRegPdst res, iRegPdst mem, iRegPsrc oldval, i
   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
   effect(TEMP_DEF res, TEMP tmp, KILL cr0);
 
-  predicate((UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong)
+  predicate((UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0)
             && (
               ((CompareAndSwapNode*)n)->order() == MemNode::acquire
               || ((CompareAndSwapNode*)n)->order() == MemNode::seqcst
@@ -272,7 +279,7 @@ instruct zCompareAndExchangeP_acq(iRegPdst res, iRegPdst mem, iRegPsrc oldval, i
   ins_encode %{
     z_compare_and_exchange(_masm, this,
                            $res$$Register, $mem$$Register, $oldval$$Register, $newval$$Register, $tmp$$Register,
-                           false /* weak */, true /* acquire */);
+                           true /* acquire */);
   %}
   ins_pipe(pipe_class_default);
 %}
@@ -281,12 +288,14 @@ instruct zGetAndSetP(iRegPdst res, iRegPdst mem, iRegPsrc newval, iRegPdst tmp,
   match(Set res (GetAndSetP mem newval));
   effect(TEMP_DEF res, TEMP tmp, KILL cr0);
 
-  predicate(UseZGC && n->as_LoadStore()->barrier_data() != 0);
+  predicate(UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0);
 
   format %{ "GetAndSetP $res, $mem, $newval" %}
   ins_encode %{
-    __ getandsetd($res$$Register, $newval$$Register, $mem$$Register, MacroAssembler::cmpxchgx_hint_atomic_update());
-    z_load_barrier(_masm, this, Address(noreg, (intptr_t) 0), $res$$Register, $tmp$$Register, barrier_data());
+    Register rnew_zpointer = $tmp$$Register, result = $res$$Register;
+    z_store_barrier(_masm, this, $mem$$Register, 0, $newval$$Register, rnew_zpointer, true /* is_atomic */);
+    __ getandsetd(result, rnew_zpointer, $mem$$Register, MacroAssembler::cmpxchgx_hint_atomic_update());
+    z_uncolor(_masm, result);
 
     if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
       __ isync();
diff --git a/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp b/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp
index 69972bf9eac..f81d49684c9 100644
--- a/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp
+++ b/src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp
@@ -354,7 +354,7 @@ inline void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorat
                                             Register tmp1, Register tmp2, Register tmp3,
                                             MacroAssembler::PreservationLevel preservation_level) {
   assert((decorators & ~(AS_RAW | IN_HEAP | IN_NATIVE | IS_ARRAY | IS_NOT_NULL |
-                         ON_UNKNOWN_OOP_REF)) == 0, "unsupported decorator");
+                         ON_UNKNOWN_OOP_REF | IS_DEST_UNINITIALIZED)) == 0, "unsupported decorator");
   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
   bool as_raw = (decorators & AS_RAW) != 0;
   decorators = AccessInternal::decorator_fixup(decorators, type);
diff --git a/src/hotspot/cpu/ppc/relocInfo_ppc.hpp b/src/hotspot/cpu/ppc/relocInfo_ppc.hpp
index f3c14c800c3..e7a23d2de91 100644
--- a/src/hotspot/cpu/ppc/relocInfo_ppc.hpp
+++ b/src/hotspot/cpu/ppc/relocInfo_ppc.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2018 SAP SE. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
@@ -39,7 +39,8 @@
     format_width       =  0
 #else
     // Except narrow oops in 64-bits VM.
-    format_width       =  1
+    // Must be at least 2 for ZGC GC barrier patching.
+    format_width       =  2
 #endif
   };
 
diff --git a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
index a20baaee5c8..ad81348e7fa 100644
--- a/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
+++ b/src/hotspot/cpu/ppc/stubGenerator_ppc.cpp
@@ -47,6 +47,10 @@
 #include "runtime/vm_version.hpp"
 #include "utilities/align.hpp"
 #include "utilities/powerOfTwo.hpp"
+#if INCLUDE_ZGC
+#include "gc/x/xBarrierSetAssembler.hpp"
+#include "gc/z/zBarrierSetAssembler.hpp"
+#endif
 
 // Declaration and definition of StubGenerator (no .hpp file).
 // For a more detailed description of the stub routine structure
@@ -61,9 +65,9 @@
 #endif
 
 #if defined(ABI_ELFv2)
-#define STUB_ENTRY(name) StubRoutines::name()
+#define STUB_ENTRY(name) StubRoutines::name
 #else
-#define STUB_ENTRY(name) ((FunctionDescriptor*)StubRoutines::name())->entry()
+#define STUB_ENTRY(name) ((FunctionDescriptor*)StubRoutines::name)->entry()
 #endif
 
 class StubGenerator: public StubCodeGenerator {
@@ -1182,8 +1186,8 @@ class StubGenerator: public StubCodeGenerator {
     Register tmp3 = R8_ARG6;
 
     address nooverlap_target = aligned ?
-      STUB_ENTRY(arrayof_jbyte_disjoint_arraycopy) :
-      STUB_ENTRY(jbyte_disjoint_arraycopy);
+      STUB_ENTRY(arrayof_jbyte_disjoint_arraycopy()) :
+      STUB_ENTRY(jbyte_disjoint_arraycopy());
 
     array_overlap_test(nooverlap_target, 0);
     // Do reverse copy. We assume the case of actual overlap is rare enough
@@ -1454,8 +1458,8 @@ class StubGenerator: public StubCodeGenerator {
     Register tmp3 = R8_ARG6;
 
     address nooverlap_target = aligned ?
-      STUB_ENTRY(arrayof_jshort_disjoint_arraycopy) :
-      STUB_ENTRY(jshort_disjoint_arraycopy);
+      STUB_ENTRY(arrayof_jshort_disjoint_arraycopy()) :
+      STUB_ENTRY(jshort_disjoint_arraycopy());
 
     array_overlap_test(nooverlap_target, 1);
 
@@ -1767,8 +1771,8 @@ class StubGenerator: public StubCodeGenerator {
     address start = __ function_entry();
     assert_positive_int(R5_ARG3);
     address nooverlap_target = aligned ?
-      STUB_ENTRY(arrayof_jint_disjoint_arraycopy) :
-      STUB_ENTRY(jint_disjoint_arraycopy);
+      STUB_ENTRY(arrayof_jint_disjoint_arraycopy()) :
+      STUB_ENTRY(jint_disjoint_arraycopy());
 
     array_overlap_test(nooverlap_target, 2);
     {
@@ -2024,8 +2028,8 @@ class StubGenerator: public StubCodeGenerator {
     address start = __ function_entry();
     assert_positive_int(R5_ARG3);
     address nooverlap_target = aligned ?
-      STUB_ENTRY(arrayof_jlong_disjoint_arraycopy) :
-      STUB_ENTRY(jlong_disjoint_arraycopy);
+      STUB_ENTRY(arrayof_jlong_disjoint_arraycopy()) :
+      STUB_ENTRY(jlong_disjoint_arraycopy());
 
     array_overlap_test(nooverlap_target, 3);
     {
@@ -2054,8 +2058,10 @@ class StubGenerator: public StubCodeGenerator {
     address start = __ function_entry();
     assert_positive_int(R5_ARG3);
     address nooverlap_target = aligned ?
-      STUB_ENTRY(arrayof_oop_disjoint_arraycopy) :
-      STUB_ENTRY(oop_disjoint_arraycopy);
+      STUB_ENTRY(arrayof_oop_disjoint_arraycopy(dest_uninitialized)) :
+      STUB_ENTRY(oop_disjoint_arraycopy(dest_uninitialized));
+
+    array_overlap_test(nooverlap_target, UseCompressedOops ? 2 : 3);
 
     DecoratorSet decorators = IN_HEAP | IS_ARRAY;
     if (dest_uninitialized) {
@@ -2069,10 +2075,14 @@ class StubGenerator: public StubCodeGenerator {
     bs->arraycopy_prologue(_masm, decorators, T_OBJECT, R3_ARG1, R4_ARG2, R5_ARG3, noreg, noreg);
 
     if (UseCompressedOops) {
-      array_overlap_test(nooverlap_target, 2);
       generate_conjoint_int_copy_core(aligned);
     } else {
-      array_overlap_test(nooverlap_target, 3);
+#if INCLUDE_ZGC
+      if (UseZGC && ZGenerational) {
+        ZBarrierSetAssembler *zbs = (ZBarrierSetAssembler*)bs;
+        zbs->generate_conjoint_oop_copy(_masm, dest_uninitialized);
+      } else
+#endif
       generate_conjoint_long_copy_core(aligned);
     }
 
@@ -2110,6 +2120,12 @@ class StubGenerator: public StubCodeGenerator {
     if (UseCompressedOops) {
       generate_disjoint_int_copy_core(aligned);
     } else {
+#if INCLUDE_ZGC
+      if (UseZGC && ZGenerational) {
+        ZBarrierSetAssembler *zbs = (ZBarrierSetAssembler*)bs;
+        zbs->generate_disjoint_oop_copy(_masm, dest_uninitialized);
+      } else
+#endif
       generate_disjoint_long_copy_core(aligned);
     }
 
@@ -2222,6 +2238,13 @@ class StubGenerator: public StubCodeGenerator {
       __ stw(R10_oop, R8_offset, R4_to);
     } else {
       __ bind(store_null);
+#if INCLUDE_ZGC
+      if (UseZGC && ZGenerational) {
+        __ store_heap_oop(R10_oop, R8_offset, R4_to, R11_scratch1, R12_tmp, noreg,
+                          MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS,
+                          dest_uninitialized ? IS_DEST_UNINITIALIZED : 0);
+      } else
+#endif
       __ std(R10_oop, R8_offset, R4_to);
     }
 
@@ -2231,6 +2254,14 @@ class StubGenerator: public StubCodeGenerator {
 
     // ======== loop entry is here ========
     __ bind(load_element);
+#if INCLUDE_ZGC
+    if (UseZGC && ZGenerational) {
+      __ load_heap_oop(R10_oop, R8_offset, R3_from,
+                       R11_scratch1, R12_tmp,
+                       MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS,
+                       0, &store_null);
+    } else
+#endif
     __ load_heap_oop(R10_oop, R8_offset, R3_from,
                      R11_scratch1, R12_tmp,
                      MacroAssembler::PRESERVATION_FRAME_LR_GP_REGS,
@@ -3136,18 +3167,18 @@ class StubGenerator: public StubCodeGenerator {
     StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", true);
 
     StubRoutines::_unsafe_arraycopy  = generate_unsafe_copy("unsafe_arraycopy",
-                                                            STUB_ENTRY(jbyte_arraycopy),
-                                                            STUB_ENTRY(jshort_arraycopy),
-                                                            STUB_ENTRY(jint_arraycopy),
-                                                            STUB_ENTRY(jlong_arraycopy));
+                                                            STUB_ENTRY(jbyte_arraycopy()),
+                                                            STUB_ENTRY(jshort_arraycopy()),
+                                                            STUB_ENTRY(jint_arraycopy()),
+                                                            STUB_ENTRY(jlong_arraycopy()));
     StubRoutines::_generic_arraycopy = generate_generic_copy("generic_arraycopy",
-                                                             STUB_ENTRY(jbyte_arraycopy),
-                                                             STUB_ENTRY(jshort_arraycopy),
-                                                             STUB_ENTRY(jint_arraycopy),
-                                                             STUB_ENTRY(oop_arraycopy),
-                                                             STUB_ENTRY(oop_disjoint_arraycopy),
-                                                             STUB_ENTRY(jlong_arraycopy),
-                                                             STUB_ENTRY(checkcast_arraycopy));
+                                                             STUB_ENTRY(jbyte_arraycopy()),
+                                                             STUB_ENTRY(jshort_arraycopy()),
+                                                             STUB_ENTRY(jint_arraycopy()),
+                                                             STUB_ENTRY(oop_arraycopy()),
+                                                             STUB_ENTRY(oop_disjoint_arraycopy()),
+                                                             STUB_ENTRY(jlong_arraycopy()),
+                                                             STUB_ENTRY(checkcast_arraycopy()));
 
     // fill routines
 #ifdef COMPILER2
diff --git a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
index 161a8a7376f..b514f8e115e 100644
--- a/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/c1_LIRAssembler_riscv.cpp
@@ -858,7 +858,7 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
       __ decode_heap_oop(dest->as_register());
     }
 
-    if (!UseZGC) {
+    if (!(UseZGC && !ZGenerational)) {
       // Load barrier has not yet been applied, so ZGC can't verify the oop here
       __ verify_oop(dest->as_register());
     }
@@ -1264,10 +1264,13 @@ void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
     if (UseCompressedOops) {
       Register tmp1 = op->tmp1()->as_register();
       assert(op->tmp1()->is_valid(), "must be");
+      Register tmp2 = op->tmp2()->as_register();
+      assert(op->tmp2()->is_valid(), "must be");
+
       __ encode_heap_oop(tmp1, cmpval);
       cmpval = tmp1;
-      __ encode_heap_oop(t1, newval);
-      newval = t1;
+      __ encode_heap_oop(tmp2, newval);
+      newval = tmp2;
       caswu(addr, newval, cmpval);
     } else {
       casl(addr, newval, cmpval);
@@ -1277,6 +1280,11 @@ void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
   } else {
     casl(addr, newval, cmpval);
   }
+
+  if (op->result_opr()->is_valid()) {
+    assert(op->result_opr()->is_register(), "need a register");
+    __ mv(as_reg(op->result_opr()), t0); // cas result in t0, and 0 for success
+  }
 }
 
 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
diff --git a/src/hotspot/cpu/riscv/gc/x/xBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/x/xBarrierSetAssembler_riscv.cpp
new file mode 100644
index 00000000000..c48eac944c1
--- /dev/null
+++ b/src/hotspot/cpu/riscv/gc/x/xBarrierSetAssembler_riscv.cpp
@@ -0,0 +1,458 @@
+/*
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "asm/macroAssembler.inline.hpp"
+#include "code/codeBlob.hpp"
+#include "code/vmreg.inline.hpp"
+#include "gc/x/xBarrier.inline.hpp"
+#include "gc/x/xBarrierSet.hpp"
+#include "gc/x/xBarrierSetAssembler.hpp"
+#include "gc/x/xBarrierSetRuntime.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+#include "memory/resourceArea.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "utilities/macros.hpp"
+#ifdef COMPILER1
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/x/c1/xBarrierSetC1.hpp"
+#endif // COMPILER1
+#ifdef COMPILER2
+#include "gc/x/c2/xBarrierSetC2.hpp"
+#endif // COMPILER2
+
+#ifdef PRODUCT
+#define BLOCK_COMMENT(str) /* nothing */
+#else
+#define BLOCK_COMMENT(str) __ block_comment(str)
+#endif
+
+#undef __
+#define __ masm->
+
+void XBarrierSetAssembler::load_at(MacroAssembler* masm,
+                                   DecoratorSet decorators,
+                                   BasicType type,
+                                   Register dst,
+                                   Address src,
+                                   Register tmp1,
+                                   Register tmp2) {
+  if (!XBarrierSet::barrier_needed(decorators, type)) {
+    // Barrier not needed
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp2);
+    return;
+  }
+
+  assert_different_registers(t1, src.base());
+  assert_different_registers(t0, t1, dst);
+
+  Label done;
+
+  // Load bad mask into temp register.
+  __ la(t0, src);
+  __ ld(t1, address_bad_mask_from_thread(xthread));
+  __ ld(dst, Address(t0));
+
+  // Test reference against bad mask. If mask bad, then we need to fix it up.
+  __ andr(t1, dst, t1);
+  __ beqz(t1, done);
+
+  __ enter();
+
+  __ push_call_clobbered_registers_except(RegSet::of(dst));
+
+  if (c_rarg0 != dst) {
+    __ mv(c_rarg0, dst);
+  }
+
+  __ mv(c_rarg1, t0);
+
+  __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
+
+  // Make sure dst has the return value.
+  if (dst != x10) {
+    __ mv(dst, x10);
+  }
+
+  __ pop_call_clobbered_registers_except(RegSet::of(dst));
+  __ leave();
+
+  __ bind(done);
+}
+
+#ifdef ASSERT
+
+void XBarrierSetAssembler::store_at(MacroAssembler* masm,
+                                    DecoratorSet decorators,
+                                    BasicType type,
+                                    Address dst,
+                                    Register val,
+                                    Register tmp1,
+                                    Register tmp2,
+                                    Register tmp3) {
+  // Verify value
+  if (is_reference_type(type)) {
+    // Note that src could be noreg, which means we
+    // are storing null and can skip verification.
+    if (val != noreg) {
+      Label done;
+
+      // tmp1, tmp2 and tmp3 are often set to noreg.
+      RegSet savedRegs = RegSet::of(t0);
+      __ push_reg(savedRegs, sp);
+
+      __ ld(t0, address_bad_mask_from_thread(xthread));
+      __ andr(t0, val, t0);
+      __ beqz(t0, done);
+      __ stop("Verify oop store failed");
+      __ should_not_reach_here();
+      __ bind(done);
+      __ pop_reg(savedRegs, sp);
+    }
+  }
+
+  // Store value
+  BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, noreg);
+}
+
+#endif // ASSERT
+
+void XBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
+                                              DecoratorSet decorators,
+                                              bool is_oop,
+                                              Register src,
+                                              Register dst,
+                                              Register count,
+                                              RegSet saved_regs) {
+  if (!is_oop) {
+    // Barrier not needed
+    return;
+  }
+
+  BLOCK_COMMENT("XBarrierSetAssembler::arraycopy_prologue {");
+
+  assert_different_registers(src, count, t0);
+
+  __ push_reg(saved_regs, sp);
+
+  if (count == c_rarg0 && src == c_rarg1) {
+    // exactly backwards!!
+    __ xorr(c_rarg0, c_rarg0, c_rarg1);
+    __ xorr(c_rarg1, c_rarg0, c_rarg1);
+    __ xorr(c_rarg0, c_rarg0, c_rarg1);
+  } else {
+    __ mv(c_rarg0, src);
+    __ mv(c_rarg1, count);
+  }
+
+  __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2);
+
+  __ pop_reg(saved_regs, sp);
+
+  BLOCK_COMMENT("} XBarrierSetAssembler::arraycopy_prologue");
+}
+
+void XBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
+                                                         Register jni_env,
+                                                         Register robj,
+                                                         Register tmp,
+                                                         Label& slowpath) {
+  BLOCK_COMMENT("XBarrierSetAssembler::try_resolve_jobject_in_native {");
+
+  assert_different_registers(jni_env, robj, tmp);
+
+  // Resolve jobject
+  BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath);
+
+  // Compute the offset of address bad mask from the field of jni_environment
+  long int bad_mask_relative_offset = (long int) (in_bytes(XThreadLocalData::address_bad_mask_offset()) -
+                                                  in_bytes(JavaThread::jni_environment_offset()));
+
+  // Load the address bad mask
+  __ ld(tmp, Address(jni_env, bad_mask_relative_offset));
+
+  // Check address bad mask
+  __ andr(tmp, robj, tmp);
+  __ bnez(tmp, slowpath);
+
+  BLOCK_COMMENT("} XBarrierSetAssembler::try_resolve_jobject_in_native");
+}
+
+#ifdef COMPILER2
+
+OptoReg::Name XBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
+  if (!OptoReg::is_reg(opto_reg)) {
+    return OptoReg::Bad;
+  }
+
+  const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+  if (vm_reg->is_FloatRegister()) {
+    return opto_reg & ~1;
+  }
+
+  return opto_reg;
+}
+
+#undef __
+#define __ _masm->
+
+class XSaveLiveRegisters {
+private:
+  MacroAssembler* const _masm;
+  RegSet                _gp_regs;
+  FloatRegSet           _fp_regs;
+  VectorRegSet          _vp_regs;
+
+public:
+  void initialize(XLoadBarrierStubC2* stub) {
+    // Record registers that needs to be saved/restored
+    RegMaskIterator rmi(stub->live());
+    while (rmi.has_next()) {
+      const OptoReg::Name opto_reg = rmi.next();
+      if (OptoReg::is_reg(opto_reg)) {
+        const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+        if (vm_reg->is_Register()) {
+          _gp_regs += RegSet::of(vm_reg->as_Register());
+        } else if (vm_reg->is_FloatRegister()) {
+          _fp_regs += FloatRegSet::of(vm_reg->as_FloatRegister());
+        } else if (vm_reg->is_VectorRegister()) {
+          const VMReg vm_reg_base = OptoReg::as_VMReg(opto_reg & ~(VectorRegister::max_slots_per_register - 1));
+          _vp_regs += VectorRegSet::of(vm_reg_base->as_VectorRegister());
+        } else {
+          fatal("Unknown register type");
+        }
+      }
+    }
+
+    // Remove C-ABI SOE registers, tmp regs and _ref register that will be updated
+    _gp_regs -= RegSet::range(x18, x27) + RegSet::of(x2) + RegSet::of(x8, x9) + RegSet::of(x5, stub->ref());
+  }
+
+  XSaveLiveRegisters(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
+      _masm(masm),
+      _gp_regs(),
+      _fp_regs(),
+      _vp_regs() {
+    // Figure out what registers to save/restore
+    initialize(stub);
+
+    // Save registers
+    __ push_reg(_gp_regs, sp);
+    __ push_fp(_fp_regs, sp);
+    __ push_v(_vp_regs, sp);
+  }
+
+  ~XSaveLiveRegisters() {
+    // Restore registers
+    __ pop_v(_vp_regs, sp);
+    __ pop_fp(_fp_regs, sp);
+    __ pop_reg(_gp_regs, sp);
+  }
+};
+
+class XSetupArguments {
+private:
+  MacroAssembler* const _masm;
+  const Register        _ref;
+  const Address         _ref_addr;
+
+public:
+  XSetupArguments(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
+      _masm(masm),
+      _ref(stub->ref()),
+      _ref_addr(stub->ref_addr()) {
+
+    // Setup arguments
+    if (_ref_addr.base() == noreg) {
+      // No self healing
+      if (_ref != c_rarg0) {
+        __ mv(c_rarg0, _ref);
+      }
+      __ mv(c_rarg1, zr);
+    } else {
+      // Self healing
+      if (_ref == c_rarg0) {
+        // _ref is already at correct place
+        __ la(c_rarg1, _ref_addr);
+      } else if (_ref != c_rarg1) {
+        // _ref is in wrong place, but not in c_rarg1, so fix it first
+        __ la(c_rarg1, _ref_addr);
+        __ mv(c_rarg0, _ref);
+      } else if (_ref_addr.base() != c_rarg0) {
+        assert(_ref == c_rarg1, "Mov ref first, vacating c_rarg0");
+        __ mv(c_rarg0, _ref);
+        __ la(c_rarg1, _ref_addr);
+      } else {
+        assert(_ref == c_rarg1, "Need to vacate c_rarg1 and _ref_addr is using c_rarg0");
+        if (_ref_addr.base() == c_rarg0) {
+          __ mv(t1, c_rarg1);
+          __ la(c_rarg1, _ref_addr);
+          __ mv(c_rarg0, t1);
+        } else {
+          ShouldNotReachHere();
+        }
+      }
+    }
+  }
+
+  ~XSetupArguments() {
+    // Transfer result
+    if (_ref != x10) {
+      __ mv(_ref, x10);
+    }
+  }
+};
+
+#undef __
+#define __ masm->
+
+void XBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, XLoadBarrierStubC2* stub) const {
+  BLOCK_COMMENT("XLoadBarrierStubC2");
+
+  // Stub entry
+  __ bind(*stub->entry());
+
+  {
+    XSaveLiveRegisters save_live_registers(masm, stub);
+    XSetupArguments setup_arguments(masm, stub);
+
+    Address target(stub->slow_path());
+    __ relocate(target.rspec(), [&] {
+      int32_t offset;
+      __ la_patchable(t0, target, offset);
+      __ jalr(x1, t0, offset);
+    });
+  }
+
+  // Stub exit
+  __ j(*stub->continuation());
+}
+
+#endif // COMPILER2
+
+#ifdef COMPILER1
+#undef __
+#define __ ce->masm()->
+
+void XBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
+                                                         LIR_Opr ref) const {
+  assert_different_registers(xthread, ref->as_register(), t1);
+  __ ld(t1, address_bad_mask_from_thread(xthread));
+  __ andr(t1, t1, ref->as_register());
+}
+
+void XBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
+                                                         XLoadBarrierStubC1* stub) const {
+  // Stub entry
+  __ bind(*stub->entry());
+
+  Register ref = stub->ref()->as_register();
+  Register ref_addr = noreg;
+  Register tmp = noreg;
+
+  if (stub->tmp()->is_valid()) {
+    // Load address into tmp register
+    ce->leal(stub->ref_addr(), stub->tmp());
+    ref_addr = tmp = stub->tmp()->as_pointer_register();
+  } else {
+    // Address already in register
+    ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
+  }
+
+  assert_different_registers(ref, ref_addr, noreg);
+
+  // Save x10 unless it is the result or tmp register
+  // Set up SP to accommodate parameters and maybe x10.
+  if (ref != x10 && tmp != x10) {
+    __ sub(sp, sp, 32);
+    __ sd(x10, Address(sp, 16));
+  } else {
+    __ sub(sp, sp, 16);
+  }
+
+  // Setup arguments and call runtime stub
+  ce->store_parameter(ref_addr, 1);
+  ce->store_parameter(ref, 0);
+
+  __ far_call(stub->runtime_stub());
+
+  // Verify result
+  __ verify_oop(x10);
+
+
+  // Move result into place
+  if (ref != x10) {
+    __ mv(ref, x10);
+  }
+
+  // Restore x10 unless it is the result or tmp register
+  if (ref != x10 && tmp != x10) {
+    __ ld(x10, Address(sp, 16));
+    __ add(sp, sp, 32);
+  } else {
+    __ add(sp, sp, 16);
+  }
+
+  // Stub exit
+  __ j(*stub->continuation());
+}
+
+#undef __
+#define __ sasm->
+
+void XBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
+                                                                 DecoratorSet decorators) const {
+  __ prologue("zgc_load_barrier stub", false);
+
+  __ push_call_clobbered_registers_except(RegSet::of(x10));
+
+  // Setup arguments
+  __ load_parameter(0, c_rarg0);
+  __ load_parameter(1, c_rarg1);
+
+  __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
+
+  __ pop_call_clobbered_registers_except(RegSet::of(x10));
+
+  __ epilogue();
+}
+
+#endif // COMPILER1
+
+#undef __
+#define __ masm->
+
+void XBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
+  // Check if mask is good.
+  // verifies that XAddressBadMask & obj == 0
+  __ ld(tmp2, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
+  __ andr(tmp1, obj, tmp2);
+  __ bnez(tmp1, error);
+
+  BarrierSetAssembler::check_oop(masm, obj, tmp1, tmp2, error);
+}
+
+#undef __
diff --git a/src/hotspot/cpu/riscv/gc/x/xBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/x/xBarrierSetAssembler_riscv.hpp
new file mode 100644
index 00000000000..cbf5077999b
--- /dev/null
+++ b/src/hotspot/cpu/riscv/gc/x/xBarrierSetAssembler_riscv.hpp
@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef CPU_RISCV_GC_X_XBARRIERSETASSEMBLER_RISCV_HPP
+#define CPU_RISCV_GC_X_XBARRIERSETASSEMBLER_RISCV_HPP
+
+#include "code/vmreg.hpp"
+#include "oops/accessDecorators.hpp"
+#ifdef COMPILER2
+#include "opto/optoreg.hpp"
+#endif // COMPILER2
+
+#ifdef COMPILER1
+class LIR_Assembler;
+class LIR_Opr;
+class StubAssembler;
+#endif // COMPILER1
+
+#ifdef COMPILER2
+class Node;
+#endif // COMPILER2
+
+#ifdef COMPILER1
+class XLoadBarrierStubC1;
+#endif // COMPILER1
+
+#ifdef COMPILER2
+class XLoadBarrierStubC2;
+#endif // COMPILER2
+
+class XBarrierSetAssembler : public XBarrierSetAssemblerBase {
+public:
+  virtual void load_at(MacroAssembler* masm,
+                       DecoratorSet decorators,
+                       BasicType type,
+                       Register dst,
+                       Address src,
+                       Register tmp1,
+                       Register tmp2);
+
+#ifdef ASSERT
+  virtual void store_at(MacroAssembler* masm,
+                        DecoratorSet decorators,
+                        BasicType type,
+                        Address dst,
+                        Register val,
+                        Register tmp1,
+                        Register tmp2,
+                        Register tmp3);
+#endif // ASSERT
+
+  virtual void arraycopy_prologue(MacroAssembler* masm,
+                                  DecoratorSet decorators,
+                                  bool is_oop,
+                                  Register src,
+                                  Register dst,
+                                  Register count,
+                                  RegSet saved_regs);
+
+  virtual void try_resolve_jobject_in_native(MacroAssembler* masm,
+                                             Register jni_env,
+                                             Register robj,
+                                             Register tmp,
+                                             Label& slowpath);
+
+  virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
+
+#ifdef COMPILER1
+  void generate_c1_load_barrier_test(LIR_Assembler* ce,
+                                     LIR_Opr ref) const;
+
+  void generate_c1_load_barrier_stub(LIR_Assembler* ce,
+                                     XLoadBarrierStubC1* stub) const;
+
+  void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
+                                             DecoratorSet decorators) const;
+#endif // COMPILER1
+
+#ifdef COMPILER2
+  OptoReg::Name refine_register(const Node* node,
+                                OptoReg::Name opto_reg);
+
+  void generate_c2_load_barrier_stub(MacroAssembler* masm,
+                                     XLoadBarrierStubC2* stub) const;
+#endif // COMPILER2
+
+  void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
+};
+
+#endif // CPU_RISCV_GC_X_XBARRIERSETASSEMBLER_RISCV_HPP
diff --git a/src/hotspot/cpu/riscv/gc/z/zGlobals_riscv.cpp b/src/hotspot/cpu/riscv/gc/x/xGlobals_riscv.cpp
similarity index 98%
rename from src/hotspot/cpu/riscv/gc/z/zGlobals_riscv.cpp
rename to src/hotspot/cpu/riscv/gc/x/xGlobals_riscv.cpp
index ddf88da646a..602dab56747 100644
--- a/src/hotspot/cpu/riscv/gc/z/zGlobals_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/x/xGlobals_riscv.cpp
@@ -26,7 +26,7 @@
 #include "precompiled.hpp"
 #include "gc/shared/gcLogPrecious.hpp"
 #include "gc/shared/gc_globals.hpp"
-#include "gc/z/zGlobals.hpp"
+#include "gc/x/xGlobals.hpp"
 #include "runtime/globals.hpp"
 #include "runtime/os.hpp"
 #include "utilities/globalDefinitions.hpp"
@@ -198,15 +198,15 @@ static size_t probe_valid_max_address_bit() {
 #endif // LINUX
 }
 
-size_t ZPlatformAddressOffsetBits() {
+size_t XPlatformAddressOffsetBits() {
   const static size_t valid_max_address_offset_bits = probe_valid_max_address_bit() + 1;
   const size_t max_address_offset_bits = valid_max_address_offset_bits - 3;
   const size_t min_address_offset_bits = max_address_offset_bits - 2;
-  const size_t address_offset = round_up_power_of_2(MaxHeapSize * ZVirtualToPhysicalRatio);
+  const size_t address_offset = round_up_power_of_2(MaxHeapSize * XVirtualToPhysicalRatio);
   const size_t address_offset_bits = log2i_exact(address_offset);
   return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
 }
 
-size_t ZPlatformAddressMetadataShift() {
-  return ZPlatformAddressOffsetBits();
+size_t XPlatformAddressMetadataShift() {
+  return XPlatformAddressOffsetBits();
 }
diff --git a/src/hotspot/cpu/riscv/gc/x/xGlobals_riscv.hpp b/src/hotspot/cpu/riscv/gc/x/xGlobals_riscv.hpp
new file mode 100644
index 00000000000..836dc7aac0d
--- /dev/null
+++ b/src/hotspot/cpu/riscv/gc/x/xGlobals_riscv.hpp
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef CPU_RISCV_GC_X_XGLOBALS_RISCV_HPP
+#define CPU_RISCV_GC_X_XGLOBALS_RISCV_HPP
+
+const size_t XPlatformHeapViews        = 3;
+const size_t XPlatformCacheLineSize    = 64;
+
+size_t XPlatformAddressOffsetBits();
+size_t XPlatformAddressMetadataShift();
+
+#endif // CPU_RISCV_GC_X_XGLOBALS_RISCV_HPP
diff --git a/src/hotspot/cpu/riscv/gc/x/x_riscv64.ad b/src/hotspot/cpu/riscv/gc/x/x_riscv64.ad
new file mode 100644
index 00000000000..73d337bc484
--- /dev/null
+++ b/src/hotspot/cpu/riscv/gc/x/x_riscv64.ad
@@ -0,0 +1,233 @@
+//
+// Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+
+source_hpp %{
+
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/c2/xBarrierSetC2.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+
+%}
+
+source %{
+
+static void x_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, int barrier_data) {
+  if (barrier_data == XLoadBarrierElided) {
+    return;
+  }
+  XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
+  __ ld(tmp, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
+  __ andr(tmp, tmp, ref);
+  __ bnez(tmp, *stub->entry(), true /* far */);
+  __ bind(*stub->continuation());
+}
+
+static void x_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
+  XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, XLoadBarrierStrong);
+  __ j(*stub->entry());
+  __ bind(*stub->continuation());
+}
+
+%}
+
+// Load Pointer
+instruct xLoadP(iRegPNoSp dst, memory mem)
+%{
+  match(Set dst (LoadP mem));
+  predicate(UseZGC && !ZGenerational && (n->as_Load()->barrier_data() != 0));
+  effect(TEMP dst);
+
+  ins_cost(4 * DEFAULT_COST);
+
+  format %{ "ld  $dst, $mem, #@zLoadP" %}
+
+  ins_encode %{
+    const Address ref_addr (as_Register($mem$$base), $mem$$disp);
+    __ ld($dst$$Register, ref_addr);
+    x_load_barrier(_masm, this, ref_addr, $dst$$Register, t0 /* tmp */, barrier_data());
+  %}
+
+  ins_pipe(iload_reg_mem);
+%}
+
+instruct xCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+  predicate(UseZGC && !ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
+  effect(KILL cr, TEMP_DEF res);
+
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  format %{ "cmpxchg $mem, $oldval, $newval, #@zCompareAndSwapP\n\t"
+            "mv $res, $res == $oldval" %}
+
+  ins_encode %{
+    Label failed;
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
+               Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register,
+               true /* result_as_bool */);
+    __ beqz($res$$Register, failed);
+    __ mv(t0, $oldval$$Register);
+    __ bind(failed);
+    if (barrier_data() != XLoadBarrierElided) {
+      Label good;
+      __ ld(t1, Address(xthread, XThreadLocalData::address_bad_mask_offset()), t1 /* tmp */);
+      __ andr(t1, t1, t0);
+      __ beqz(t1, good);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), t0 /* ref */, t1 /* tmp */);
+      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
+                 Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register,
+                 true /* result_as_bool */);
+      __ bind(good);
+    }
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct xCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+  predicate(UseZGC && !ZGenerational && needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() == XLoadBarrierStrong));
+  effect(KILL cr, TEMP_DEF res);
+
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  format %{ "cmpxchg $mem, $oldval, $newval, #@zCompareAndSwapPAcq\n\t"
+            "mv $res, $res == $oldval" %}
+
+  ins_encode %{
+    Label failed;
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
+               Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register,
+               true /* result_as_bool */);
+    __ beqz($res$$Register, failed);
+    __ mv(t0, $oldval$$Register);
+    __ bind(failed);
+    if (barrier_data() != XLoadBarrierElided) {
+      Label good;
+      __ ld(t1, Address(xthread, XThreadLocalData::address_bad_mask_offset()), t1 /* tmp */);
+      __ andr(t1, t1, t0);
+      __ beqz(t1, good);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), t0 /* ref */, t1 /* tmp */);
+      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
+                 Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register,
+                 true /* result_as_bool */);
+      __ bind(good);
+    }
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct xCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval) %{
+  match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+  predicate(UseZGC && !ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
+  effect(TEMP_DEF res);
+
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  format %{ "cmpxchg $res = $mem, $oldval, $newval, #@zCompareAndExchangeP" %}
+
+  ins_encode %{
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
+               Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register);
+    if (barrier_data() != XLoadBarrierElided) {
+      Label good;
+      __ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
+      __ andr(t0, t0, $res$$Register);
+      __ beqz(t0, good);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, t0 /* tmp */);
+      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
+                 Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register);
+      __ bind(good);
+    }
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct xCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval) %{
+  match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
+  predicate(UseZGC && !ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
+  effect(TEMP_DEF res);
+
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  format %{ "cmpxchg $res = $mem, $oldval, $newval, #@zCompareAndExchangePAcq" %}
+
+  ins_encode %{
+    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
+    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
+               Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register);
+    if (barrier_data() != XLoadBarrierElided) {
+      Label good;
+      __ ld(t0, Address(xthread, XThreadLocalData::address_bad_mask_offset()));
+      __ andr(t0, t0, $res$$Register);
+      __ beqz(t0, good);
+      x_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, t0 /* tmp */);
+      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
+                 Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register);
+      __ bind(good);
+    }
+  %}
+
+  ins_pipe(pipe_slow);
+%}
+
+instruct xGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+  match(Set prev (GetAndSetP mem newv));
+  predicate(UseZGC && !ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP_DEF prev, KILL cr);
+
+  ins_cost(2 * VOLATILE_REF_COST);
+
+  format %{ "atomic_xchg  $prev, $newv, [$mem], #@zGetAndSetP" %}
+
+  ins_encode %{
+    __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
+    x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, t0 /* tmp */, barrier_data());
+  %}
+
+  ins_pipe(pipe_serial);
+%}
+
+instruct xGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
+  match(Set prev (GetAndSetP mem newv));
+  predicate(UseZGC && !ZGenerational && needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() != 0));
+  effect(TEMP_DEF prev, KILL cr);
+
+  ins_cost(VOLATILE_REF_COST);
+
+  format %{ "atomic_xchg_acq  $prev, $newv, [$mem], #@zGetAndSetPAcq" %}
+
+  ins_encode %{
+    __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
+    x_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, t0 /* tmp */, barrier_data());
+  %}
+  ins_pipe(pipe_serial);
+%}
diff --git a/src/hotspot/cpu/riscv/gc/z/zAddress_riscv.cpp b/src/hotspot/cpu/riscv/gc/z/zAddress_riscv.cpp
new file mode 100644
index 00000000000..ef13676b02e
--- /dev/null
+++ b/src/hotspot/cpu/riscv/gc/z/zAddress_riscv.cpp
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, Huawei Technologies Co., Ltd. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/z/zAddress.hpp"
+#include "gc/z/zBarrierSetAssembler.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/os.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+#ifdef LINUX
+#include <sys/mman.h>
+#endif // LINUX
+
+// Default value if probe is not implemented for a certain platform: 128TB
+static const size_t DEFAULT_MAX_ADDRESS_BIT = 47;
+// Minimum value returned, if probing fails: 64GB
+static const size_t MINIMUM_MAX_ADDRESS_BIT = 36;
+
+static size_t probe_valid_max_address_bit() {
+#ifdef LINUX
+  size_t max_address_bit = 0;
+  const size_t page_size = os::vm_page_size();
+  for (size_t i = DEFAULT_MAX_ADDRESS_BIT; i > MINIMUM_MAX_ADDRESS_BIT; --i) {
+    const uintptr_t base_addr = ((uintptr_t) 1U) << i;
+    if (msync((void*)base_addr, page_size, MS_ASYNC) == 0) {
+      // msync suceeded, the address is valid, and maybe even already mapped.
+      max_address_bit = i;
+      break;
+    }
+    if (errno != ENOMEM) {
+      // Some error occured. This should never happen, but msync
+      // has some undefined behavior, hence ignore this bit.
+#ifdef ASSERT
+      fatal("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
+#else // ASSERT
+      log_warning_p(gc)("Received '%s' while probing the address space for the highest valid bit", os::errno_name(errno));
+#endif // ASSERT
+      continue;
+    }
+    // Since msync failed with ENOMEM, the page might not be mapped.
+    // Try to map it, to see if the address is valid.
+    void* const result_addr = mmap((void*) base_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
+    if (result_addr != MAP_FAILED) {
+      munmap(result_addr, page_size);
+    }
+    if ((uintptr_t) result_addr == base_addr) {
+      // address is valid
+      max_address_bit = i;
+      break;
+    }
+  }
+  if (max_address_bit == 0) {
+    // probing failed, allocate a very high page and take that bit as the maximum
+    const uintptr_t high_addr = ((uintptr_t) 1U) << DEFAULT_MAX_ADDRESS_BIT;
+    void* const result_addr = mmap((void*) high_addr, page_size, PROT_NONE, MAP_PRIVATE|MAP_ANONYMOUS|MAP_NORESERVE, -1, 0);
+    if (result_addr != MAP_FAILED) {
+      max_address_bit = BitsPerSize_t - count_leading_zeros((size_t) result_addr) - 1;
+      munmap(result_addr, page_size);
+    }
+  }
+  log_info_p(gc, init)("Probing address space for the highest valid bit: " SIZE_FORMAT, max_address_bit);
+  return MAX2(max_address_bit, MINIMUM_MAX_ADDRESS_BIT);
+#else // LINUX
+  return DEFAULT_MAX_ADDRESS_BIT;
+#endif // LINUX
+}
+
+size_t ZPlatformAddressOffsetBits() {
+  const static size_t valid_max_address_offset_bits = probe_valid_max_address_bit() + 1;
+  const size_t max_address_offset_bits = valid_max_address_offset_bits - 3;
+  const size_t min_address_offset_bits = max_address_offset_bits - 2;
+  const size_t address_offset = round_up_power_of_2(MaxHeapSize * ZVirtualToPhysicalRatio);
+  const size_t address_offset_bits = log2i_exact(address_offset);
+  return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
+}
+
+size_t ZPlatformAddressHeapBaseShift() {
+  return ZPlatformAddressOffsetBits();
+}
+
+void ZGlobalsPointers::pd_set_good_masks() {
+  BarrierSetAssembler::clear_patching_epoch();
+}
diff --git a/src/hotspot/cpu/riscv/gc/z/zAddress_riscv.hpp b/src/hotspot/cpu/riscv/gc/z/zAddress_riscv.hpp
new file mode 100644
index 00000000000..0ca8faf1464
--- /dev/null
+++ b/src/hotspot/cpu/riscv/gc/z/zAddress_riscv.hpp
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_RISCV_GC_Z_ZADDRESS_RISCV_HPP
+#define CPU_RISCV_GC_Z_ZADDRESS_RISCV_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+const size_t ZPointerLoadShift = 16;
+
+size_t ZPlatformAddressOffsetBits();
+size_t ZPlatformAddressHeapBaseShift();
+
+#endif // CPU_RISCV_GC_Z_ZADDRESS_RISCV_HPP
diff --git a/src/hotspot/cpu/riscv/gc/z/zAddress_riscv.inline.hpp b/src/hotspot/cpu/riscv/gc/z/zAddress_riscv.inline.hpp
new file mode 100644
index 00000000000..8992eadeb46
--- /dev/null
+++ b/src/hotspot/cpu/riscv/gc/z/zAddress_riscv.inline.hpp
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, Huawei Technologies Co., Ltd. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_RISCV_GC_Z_ZADDRESS_RISCV_INLINE_HPP
+#define CPU_RISCV_GC_Z_ZADDRESS_RISCV_INLINE_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+inline uintptr_t ZPointer::remap_bits(uintptr_t colored) {
+  return colored & ZPointerRemappedMask;
+}
+
+inline constexpr int ZPointer::load_shift_lookup(uintptr_t value) {
+  return ZPointerLoadShift;
+}
+
+#endif // CPU_RISCV_GC_Z_ZADDRESS_RISCV_INLINE_HPP
diff --git a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp
index ee2844e0180..000a6047846 100644
--- a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.cpp
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,13 +27,16 @@
 #include "asm/macroAssembler.inline.hpp"
 #include "code/codeBlob.hpp"
 #include "code/vmreg.inline.hpp"
+#include "gc/z/zAddress.hpp"
 #include "gc/z/zBarrier.inline.hpp"
 #include "gc/z/zBarrierSet.hpp"
 #include "gc/z/zBarrierSetAssembler.hpp"
 #include "gc/z/zBarrierSetRuntime.hpp"
 #include "gc/z/zThreadLocalData.hpp"
 #include "memory/resourceArea.hpp"
+#include "runtime/jniHandles.hpp"
 #include "runtime/sharedRuntime.hpp"
+#include "utilities/debug.hpp"
 #include "utilities/macros.hpp"
 #ifdef COMPILER1
 #include "c1/c1_LIRAssembler.hpp"
@@ -42,6 +45,7 @@
 #endif // COMPILER1
 #ifdef COMPILER2
 #include "gc/z/c2/zBarrierSetC2.hpp"
+#include "opto/output.hpp"
 #endif // COMPILER2
 
 #ifdef PRODUCT
@@ -53,6 +57,52 @@
 #undef __
 #define __ masm->
 
+// Helper for saving and restoring registers across a runtime call that does
+// not have any live vector registers.
+class ZRuntimeCallSpill {
+private:
+  MacroAssembler* _masm;
+  Register _result;
+
+  void save() {
+    MacroAssembler* masm = _masm;
+
+    __ enter();
+    if (_result != noreg) {
+      __ push_call_clobbered_registers_except(RegSet::of(_result));
+    } else {
+      __ push_call_clobbered_registers();
+    }
+  }
+
+  void restore() {
+    MacroAssembler* masm = _masm;
+
+    if (_result != noreg) {
+      // Make sure _result has the return value.
+      if (_result != x10) {
+        __ mv(_result, x10);
+      }
+
+      __ pop_call_clobbered_registers_except(RegSet::of(_result));
+    } else {
+      __ pop_call_clobbered_registers();
+    }
+    __ leave();
+  }
+
+public:
+  ZRuntimeCallSpill(MacroAssembler* masm, Register result)
+    : _masm(masm),
+      _result(result) {
+    save();
+  }
+
+  ~ZRuntimeCallSpill() {
+    restore();
+  }
+};
+
 void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
                                    DecoratorSet decorators,
                                    BasicType type,
@@ -66,44 +116,197 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
     return;
   }
 
-  assert_different_registers(t1, src.base());
-  assert_different_registers(t0, t1, dst);
+  assert_different_registers(tmp1, tmp2, src.base(), noreg);
+  assert_different_registers(tmp1, tmp2, dst, noreg);
+  assert_different_registers(tmp2, t0);
 
   Label done;
+  Label uncolor;
 
-  // Load bad mask into temp register.
-  __ la(t0, src);
-  __ ld(t1, address_bad_mask_from_thread(xthread));
-  __ ld(dst, Address(t0));
+  // Load bad mask into scratch register.
+  const bool on_non_strong =
+    (decorators & ON_WEAK_OOP_REF) != 0 ||
+    (decorators & ON_PHANTOM_OOP_REF) != 0;
+
+  if (on_non_strong) {
+    __ ld(tmp1, mark_bad_mask_from_thread(xthread));
+  } else {
+    __ ld(tmp1, load_bad_mask_from_thread(xthread));
+  }
+
+  __ la(tmp2, src);
+  __ ld(dst, tmp2);
 
   // Test reference against bad mask. If mask bad, then we need to fix it up.
-  __ andr(t1, dst, t1);
-  __ beqz(t1, done);
+  __ andr(tmp1, dst, tmp1);
+  __ beqz(tmp1, uncolor);
 
-  __ enter();
+  {
+    // Call VM
+    ZRuntimeCallSpill rsc(masm, dst);
 
-  __ push_call_clobbered_registers_except(RegSet::of(dst));
+    if (c_rarg0 != dst) {
+      __ mv(c_rarg0, dst);
+    }
+    __ mv(c_rarg1, tmp2);
 
-  if (c_rarg0 != dst) {
-    __ mv(c_rarg0, dst);
+    __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
   }
 
-  __ mv(c_rarg1, t0);
+  // Slow-path has already uncolored
+  __ j(done);
 
-  __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), 2);
+  __ bind(uncolor);
 
-  // Make sure dst has the return value.
-  if (dst != x10) {
-    __ mv(dst, x10);
-  }
-
-  __ pop_call_clobbered_registers_except(RegSet::of(dst));
-  __ leave();
+  // Remove the color bits
+  __ srli(dst, dst, ZPointerLoadShift);
 
   __ bind(done);
 }
 
-#ifdef ASSERT
+void ZBarrierSetAssembler::store_barrier_fast(MacroAssembler* masm,
+                                              Address ref_addr,
+                                              Register rnew_zaddress,
+                                              Register rnew_zpointer,
+                                              Register rtmp,
+                                              bool in_nmethod,
+                                              bool is_atomic,
+                                              Label& medium_path,
+                                              Label& medium_path_continuation) const {
+  assert_different_registers(ref_addr.base(), rnew_zpointer, rtmp);
+  assert_different_registers(rnew_zaddress, rnew_zpointer, rtmp);
+
+  if (in_nmethod) {
+    if (is_atomic) {
+      __ lhu(rtmp, ref_addr);
+      // Atomic operations must ensure that the contents of memory are store-good before
+      // an atomic opertion can execute.
+      // A non-relocatable object could have spurious raw null pointers in its fields after
+      // getting promoted to the old generation.
+      __ relocate(barrier_Relocation::spec(), [&] {
+        __ li16u(rnew_zpointer, barrier_Relocation::unpatched);
+      }, ZBarrierRelocationFormatStoreGoodBits);
+      __ bne(rtmp, rnew_zpointer, medium_path, true /* is_far */);
+    } else {
+      __ ld(rtmp, ref_addr);
+      // Stores on relocatable objects never need to deal with raw null pointers in fields.
+      // Raw null pointers may only exists in the young generation, as they get pruned when
+      // the object is relocated to old. And no pre-write barrier needs to perform any action
+      // in the young generation.
+      __ relocate(barrier_Relocation::spec(), [&] {
+        __ li16u(rnew_zpointer, barrier_Relocation::unpatched);
+      }, ZBarrierRelocationFormatStoreBadMask);
+      __ andr(rtmp, rtmp, rnew_zpointer);
+      __ bnez(rtmp, medium_path, true /* is_far */);
+    }
+    __ bind(medium_path_continuation);
+    __ relocate(barrier_Relocation::spec(), [&] {
+      __ li16u(rtmp, barrier_Relocation::unpatched);
+    }, ZBarrierRelocationFormatStoreGoodBits);
+    __ slli(rnew_zpointer, rnew_zaddress, ZPointerLoadShift);
+    __ orr(rnew_zpointer, rnew_zpointer, rtmp);
+  } else {
+    assert(!is_atomic, "atomic outside of nmethods not supported");
+    __ la(rtmp, ref_addr);
+    __ ld(rtmp, rtmp);
+    __ ld(rnew_zpointer, Address(xthread, ZThreadLocalData::store_bad_mask_offset()));
+    __ andr(rtmp, rtmp, rnew_zpointer);
+    __ bnez(rtmp, medium_path, true /* is_far */);
+    __ bind(medium_path_continuation);
+    if (rnew_zaddress == noreg) {
+      __ mv(rnew_zpointer, zr);
+    } else {
+      __ mv(rnew_zpointer, rnew_zaddress);
+    }
+
+    // Load the current good shift, and add the color bits
+    __ slli(rnew_zpointer, rnew_zpointer, ZPointerLoadShift);
+    __ ld(rtmp, Address(xthread, ZThreadLocalData::store_good_mask_offset()));
+    __ orr(rnew_zpointer, rnew_zpointer, rtmp);
+  }
+}
+
+static void store_barrier_buffer_add(MacroAssembler* masm,
+                                     Address ref_addr,
+                                     Register tmp1,
+                                     Register tmp2,
+                                     Label& slow_path) {
+  Address buffer(xthread, ZThreadLocalData::store_barrier_buffer_offset());
+  assert_different_registers(ref_addr.base(), tmp1, tmp2);
+
+  __ ld(tmp1, buffer);
+
+  // Combined pointer bump and check if the buffer is disabled or full
+  __ ld(tmp2, Address(tmp1, ZStoreBarrierBuffer::current_offset()));
+  __ beqz(tmp2, slow_path);
+
+  // Bump the pointer
+  __ sub(tmp2, tmp2, sizeof(ZStoreBarrierEntry));
+  __ sd(tmp2, Address(tmp1, ZStoreBarrierBuffer::current_offset()));
+
+  // Compute the buffer entry address
+  __ la(tmp2, Address(tmp2, ZStoreBarrierBuffer::buffer_offset()));
+  __ add(tmp2, tmp2, tmp1);
+
+  // Compute and log the store address
+  __ la(tmp1, ref_addr);
+  __ sd(tmp1, Address(tmp2, in_bytes(ZStoreBarrierEntry::p_offset())));
+
+  // Load and log the prev value
+  __ ld(tmp1, tmp1);
+  __ sd(tmp1, Address(tmp2, in_bytes(ZStoreBarrierEntry::prev_offset())));
+}
+
+void ZBarrierSetAssembler::store_barrier_medium(MacroAssembler* masm,
+                                                Address ref_addr,
+                                                Register rtmp1,
+                                                Register rtmp2,
+                                                Register rtmp3,
+                                                bool is_native,
+                                                bool is_atomic,
+                                                Label& medium_path_continuation,
+                                                Label& slow_path,
+                                                Label& slow_path_continuation) const {
+  assert_different_registers(ref_addr.base(), rtmp1, rtmp2, rtmp3);
+
+  // The reason to end up in the medium path is that the pre-value was not 'good'.
+  if (is_native) {
+    __ j(slow_path);
+    __ bind(slow_path_continuation);
+    __ j(medium_path_continuation);
+  } else if (is_atomic) {
+    // Atomic accesses can get to the medium fast path because the value was a
+    // raw null value. If it was not null, then there is no doubt we need to take a slow path.
+
+    __ la(rtmp2, ref_addr);
+    __ ld(rtmp1, rtmp2);
+    __ bnez(rtmp1, slow_path);
+
+    // If we get this far, we know there is a young raw null value in the field.
+    __ relocate(barrier_Relocation::spec(), [&] {
+      __ li16u(rtmp1, barrier_Relocation::unpatched);
+    }, ZBarrierRelocationFormatStoreGoodBits);
+    __ cmpxchg_weak(rtmp2, zr, rtmp1,
+                    Assembler::int64,
+                    Assembler::relaxed /* acquire */, Assembler::relaxed /* release */,
+                    rtmp3);
+    __ beqz(rtmp3, slow_path);
+    __ bind(slow_path_continuation);
+    __ j(medium_path_continuation);
+  } else {
+    // A non-atomic relocatable object wont't get to the medium fast path due to a
+    // raw null in the young generation. We only get here because the field is bad.
+    // In this path we don't need any self healing, so we can avoid a runtime call
+    // most of the time by buffering the store barrier to be applied lazily.
+    store_barrier_buffer_add(masm,
+                             ref_addr,
+                             rtmp1,
+                             rtmp2,
+                             slow_path);
+    __ bind(slow_path_continuation);
+    __ j(medium_path_continuation);
+  }
+}
 
 void ZBarrierSetAssembler::store_at(MacroAssembler* masm,
                                     DecoratorSet decorators,
@@ -113,32 +316,103 @@ void ZBarrierSetAssembler::store_at(MacroAssembler* masm,
                                     Register tmp1,
                                     Register tmp2,
                                     Register tmp3) {
-  // Verify value
-  if (is_reference_type(type)) {
-    // Note that src could be noreg, which means we
-    // are storing null and can skip verification.
-    if (val != noreg) {
-      Label done;
+  if (!ZBarrierSet::barrier_needed(decorators, type)) {
+    BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, tmp3);
+    return;
+  }
 
-      // tmp1, tmp2 and tmp3 are often set to noreg.
-      RegSet savedRegs = RegSet::of(t0);
-      __ push_reg(savedRegs, sp);
+  bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 
-      __ ld(t0, address_bad_mask_from_thread(xthread));
-      __ andr(t0, val, t0);
-      __ beqz(t0, done);
-      __ stop("Verify oop store failed");
-      __ should_not_reach_here();
-      __ bind(done);
-      __ pop_reg(savedRegs, sp);
+  assert_different_registers(val, tmp1, dst.base());
+
+  if (dest_uninitialized) {
+    if (val == noreg) {
+      __ mv(tmp1, zr);
+    } else {
+      __ mv(tmp1, val);
     }
+    // Add the color bits
+    __ slli(tmp1, tmp1, ZPointerLoadShift);
+    __ ld(tmp2, Address(xthread, ZThreadLocalData::store_good_mask_offset()));
+    __ orr(tmp1, tmp2, tmp1);
+  } else {
+    Label done;
+    Label medium;
+    Label medium_continuation;
+    Label slow;
+    Label slow_continuation;
+    store_barrier_fast(masm, dst, val, tmp1, tmp2, false, false, medium, medium_continuation);
+
+    __ j(done);
+    __ bind(medium);
+    store_barrier_medium(masm,
+                         dst,
+                         tmp1,
+                         tmp2,
+                         noreg /* tmp3 */,
+                         false /* is_native */,
+                         false /* is_atomic */,
+                         medium_continuation,
+                         slow,
+                         slow_continuation);
+
+    __ bind(slow);
+    {
+      // Call VM
+      ZRuntimeCallSpill rcs(masm, noreg);
+      __ la(c_rarg0, dst);
+      __ MacroAssembler::call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), 1);
+    }
+
+    __ j(slow_continuation);
+    __ bind(done);
   }
 
   // Store value
-  BarrierSetAssembler::store_at(masm, decorators, type, dst, val, tmp1, tmp2, noreg);
+  BarrierSetAssembler::store_at(masm, decorators, type, dst, tmp1, tmp2, tmp3, noreg);
 }
 
-#endif // ASSERT
+class ZCopyRuntimeCallSpill {
+private:
+  MacroAssembler* _masm;
+  Register _result;
+
+  void save() {
+    MacroAssembler* masm = _masm;
+
+    __ enter();
+    if (_result != noreg) {
+      __ push_call_clobbered_registers_except(RegSet::of(_result));
+    } else {
+      __ push_call_clobbered_registers();
+    }
+  }
+
+  void restore() {
+    MacroAssembler* masm = _masm;
+
+    if (_result != noreg) {
+      if (_result != x10) {
+        __ mv(_result, x10);
+      }
+      __ pop_call_clobbered_registers_except(RegSet::of(_result));
+    } else {
+      __ pop_call_clobbered_registers();
+    }
+    __ leave();
+  }
+
+public:
+  ZCopyRuntimeCallSpill(MacroAssembler* masm, Register result)
+    : _masm(masm),
+      _result(result) {
+    save();
+  }
+
+  ~ZCopyRuntimeCallSpill() {
+    restore();
+  }
+};
 
 void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
                                               DecoratorSet decorators,
@@ -147,32 +421,137 @@ void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
                                               Register dst,
                                               Register count,
                                               RegSet saved_regs) {
-  if (!is_oop) {
-    // Barrier not needed
+}
+
+static void copy_load_barrier(MacroAssembler* masm,
+                              Register ref,
+                              Address src,
+                              Register tmp) {
+  Label done;
+
+  __ ld(tmp, Address(xthread, ZThreadLocalData::load_bad_mask_offset()));
+
+  // Test reference against bad mask. If mask bad, then we need to fix it up
+  __ andr(tmp, ref, tmp);
+  __ beqz(tmp, done);
+
+  {
+    // Call VM
+    ZCopyRuntimeCallSpill rsc(masm, ref);
+
+    __ la(c_rarg1, src);
+
+    if (c_rarg0 != ref) {
+      __ mv(c_rarg0, ref);
+    }
+
+    __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(IN_HEAP | ON_STRONG_OOP_REF), 2);
+  }
+
+  // Slow-path has uncolored; revert
+  __ slli(ref, ref, ZPointerLoadShift);
+
+  __ bind(done);
+}
+
+static void copy_store_barrier(MacroAssembler* masm,
+                               Register pre_ref,
+                               Register new_ref,
+                               Address src,
+                               Register tmp1,
+                               Register tmp2) {
+  Label done;
+  Label slow;
+
+  // Test reference against bad mask. If mask bad, then we need to fix it up.
+  __ ld(tmp1, Address(xthread, ZThreadLocalData::store_bad_mask_offset()));
+  __ andr(tmp1, pre_ref, tmp1);
+  __ beqz(tmp1, done);
+
+  store_barrier_buffer_add(masm, src, tmp1, tmp2, slow);
+  __ j(done);
+
+  __ bind(slow);
+  {
+    // Call VM
+    ZCopyRuntimeCallSpill rcs(masm, noreg);
+
+    __ la(c_rarg0, src);
+
+    __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), 1);
+  }
+
+  __ bind(done);
+
+  if (new_ref != noreg) {
+    // Set store-good color, replacing whatever color was there before
+    __ ld(tmp1, Address(xthread, ZThreadLocalData::store_good_mask_offset()));
+    __ srli(new_ref, new_ref, 16);
+    __ slli(new_ref, new_ref, 16);
+    __ orr(new_ref, new_ref, tmp1);
+  }
+}
+
+void ZBarrierSetAssembler::copy_load_at(MacroAssembler* masm,
+                                        DecoratorSet decorators,
+                                        BasicType type,
+                                        size_t bytes,
+                                        Register dst,
+                                        Address src,
+                                        Register tmp) {
+  if (!is_reference_type(type)) {
+    BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst, src, noreg);
     return;
   }
 
-  BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {");
+  BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst, src, noreg);
 
-  assert_different_registers(src, count, t0);
+  assert(bytes == 8, "unsupported copy step");
+  copy_load_barrier(masm, dst, src, tmp);
 
-  __ push_reg(saved_regs, sp);
+  if ((decorators & ARRAYCOPY_CHECKCAST) != 0) {
+    __ srli(dst, dst, ZPointerLoadShift);
+  }
+}
 
-  if (count == c_rarg0 && src == c_rarg1) {
-    // exactly backwards!!
-    __ xorr(c_rarg0, c_rarg0, c_rarg1);
-    __ xorr(c_rarg1, c_rarg0, c_rarg1);
-    __ xorr(c_rarg0, c_rarg0, c_rarg1);
-  } else {
-    __ mv(c_rarg0, src);
-    __ mv(c_rarg1, count);
+void ZBarrierSetAssembler::copy_store_at(MacroAssembler* masm,
+                                         DecoratorSet decorators,
+                                         BasicType type,
+                                         size_t bytes,
+                                         Address dst,
+                                         Register src,
+                                         Register tmp1,
+                                         Register tmp2,
+                                         Register tmp3) {
+  if (!is_reference_type(type)) {
+    BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src, noreg, noreg, noreg);
+    return;
   }
 
-  __ call_VM_leaf(ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), 2);
+  if ((decorators & ARRAYCOPY_CHECKCAST) != 0) {
+    __ slli(src, src, ZPointerLoadShift);
+  }
 
-  __ pop_reg(saved_regs, sp);
+  bool is_dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
 
-  BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue");
+  assert(bytes == 8, "unsupported copy step");
+  if (is_dest_uninitialized) {
+    __ ld(tmp1, Address(xthread, ZThreadLocalData::store_good_mask_offset()));
+    __ srli(src, src, 16);
+    __ slli(src, src, 16);
+    __ orr(src, src, tmp1);
+  } else {
+    // Store barrier pre values and color new values
+    __ ld(tmp1, dst);
+    copy_store_barrier(masm, tmp1, src, dst, tmp2, tmp3);
+  }
+
+  // Store new values
+  BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src, noreg, noreg, noreg);
+}
+
+bool ZBarrierSetAssembler::supports_rvv_arraycopy() {
+  return false;
 }
 
 void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
@@ -182,25 +561,86 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
                                                          Label& slowpath) {
   BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {");
 
-  assert_different_registers(jni_env, robj, tmp);
+  Label done, tagged, weak_tagged, uncolor;
 
-  // Resolve jobject
-  BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, robj, tmp, slowpath);
+  // Test for tag
+  __ andi(tmp, robj, JNIHandles::tag_mask);
+  __ bnez(tmp, tagged);
 
-  // Compute the offset of address bad mask from the field of jni_environment
-  long int bad_mask_relative_offset = (long int) (in_bytes(ZThreadLocalData::address_bad_mask_offset()) -
-                                                  in_bytes(JavaThread::jni_environment_offset()));
+  // Resolve local handle
+  __ ld(robj, robj);
+  __ j(done);
 
-  // Load the address bad mask
-  __ ld(tmp, Address(jni_env, bad_mask_relative_offset));
+  __ bind(tagged);
 
-  // Check address bad mask
+  // Test for weak tag
+  __ andi(tmp, robj, JNIHandles::TypeTag::weak_global);
+  __ bnez(tmp, weak_tagged);
+
+  // Resolve global handle
+  __ ld(robj, Address(robj, -JNIHandles::TypeTag::global));
+  __ la(tmp, load_bad_mask_from_jni_env(jni_env));
+  __ ld(tmp, tmp);
+  __ andr(tmp, robj, tmp);
+  __ bnez(tmp, slowpath);
+  __ j(uncolor);
+
+  __ bind(weak_tagged);
+
+  // Resolve weak handle
+  __ ld(robj, Address(robj, -JNIHandles::TypeTag::weak_global));
+  __ la(tmp, mark_bad_mask_from_jni_env(jni_env));
+  __ ld(tmp, tmp);
   __ andr(tmp, robj, tmp);
   __ bnez(tmp, slowpath);
 
+  __ bind(uncolor);
+
+  // Uncolor
+  __ srli(robj, robj, ZPointerLoadShift);
+
+  __ bind(done);
+
   BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native");
 }
 
+static uint16_t patch_barrier_relocation_value(int format) {
+  switch (format) {
+    case ZBarrierRelocationFormatLoadBadMask:
+      return (uint16_t)ZPointerLoadBadMask;
+    case ZBarrierRelocationFormatMarkBadMask:
+      return (uint16_t)ZPointerMarkBadMask;
+    case ZBarrierRelocationFormatStoreGoodBits:
+      return (uint16_t)ZPointerStoreGoodMask;
+    case ZBarrierRelocationFormatStoreBadMask:
+      return (uint16_t)ZPointerStoreBadMask;
+
+    default:
+      ShouldNotReachHere();
+      return 0;
+  }
+}
+
+void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
+  const uint16_t value = patch_barrier_relocation_value(format);
+
+  int bytes;
+  switch (format) {
+    case ZBarrierRelocationFormatLoadBadMask:
+    case ZBarrierRelocationFormatMarkBadMask:
+    case ZBarrierRelocationFormatStoreGoodBits:
+    case ZBarrierRelocationFormatStoreBadMask:
+      assert(NativeInstruction::is_li16u_at(addr), "invalide zgc barrier");
+      bytes = MacroAssembler::pd_patch_instruction_size(addr, (address)(uintptr_t)value);
+      break;
+    default:
+      ShouldNotReachHere();
+  }
+
+  // A full fence is generated before icache_flush by default in invalidate_word
+  ICache::invalidate_range(addr, bytes);
+}
+
 #ifdef COMPILER2
 
 OptoReg::Name ZBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
@@ -227,7 +667,7 @@ private:
   VectorRegSet          _vp_regs;
 
 public:
-  void initialize(ZLoadBarrierStubC2* stub) {
+  void initialize(ZBarrierStubC2* stub) {
     // Record registers that needs to be saved/restored
     RegMaskIterator rmi(stub->live());
     while (rmi.has_next()) {
@@ -248,10 +688,14 @@ public:
     }
 
     // Remove C-ABI SOE registers, tmp regs and _ref register that will be updated
-    _gp_regs -= RegSet::range(x18, x27) + RegSet::of(x2) + RegSet::of(x8, x9) + RegSet::of(x5, stub->ref());
+    if (stub->result() != noreg) {
+      _gp_regs -= RegSet::range(x18, x27) + RegSet::of(x2) + RegSet::of(x8, x9) + RegSet::of(x5, stub->result());
+    } else {
+      _gp_regs -= RegSet::range(x18, x27) + RegSet::of(x2, x5) + RegSet::of(x8, x9);
+    }
   }
 
-  ZSaveLiveRegisters(MacroAssembler* masm, ZLoadBarrierStubC2* stub) :
+  ZSaveLiveRegisters(MacroAssembler* masm, ZBarrierStubC2* stub) :
       _masm(masm),
       _gp_regs(),
       _fp_regs(),
@@ -333,35 +777,111 @@ void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, Z
   BLOCK_COMMENT("ZLoadBarrierStubC2");
 
   // Stub entry
-  __ bind(*stub->entry());
+  if (!Compile::current()->output()->in_scratch_emit_size()) {
+    __ bind(*stub->entry());
+  }
 
   {
     ZSaveLiveRegisters save_live_registers(masm, stub);
     ZSetupArguments setup_arguments(masm, stub);
-
-    Address target(stub->slow_path());
-    __ relocate(target.rspec(), [&] {
-      int32_t offset;
-      __ la_patchable(t0, target, offset);
-      __ jalr(x1, t0, offset);
-    });
+    __ mv(t0, stub->slow_path());
+    __ jalr(t0);
   }
 
   // Stub exit
   __ j(*stub->continuation());
 }
 
+void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm, ZStoreBarrierStubC2* stub) const {
+  BLOCK_COMMENT("ZStoreBarrierStubC2");
+
+  // Stub entry
+  __ bind(*stub->entry());
+
+  Label slow;
+  Label slow_continuation;
+  store_barrier_medium(masm,
+                       stub->ref_addr(),
+                       stub->new_zpointer(),
+                       t1,
+                       t0,
+                       stub->is_native(),
+                       stub->is_atomic(),
+                       *stub->continuation(),
+                       slow,
+                       slow_continuation);
+
+  __ bind(slow);
+
+  {
+    ZSaveLiveRegisters save_live_registers(masm, stub);
+    __ la(c_rarg0, stub->ref_addr());
+
+    if (stub->is_native()) {
+      __ la(t0, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr()));
+    } else if (stub->is_atomic()) {
+      __ la(t0, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr()));
+    } else {
+      __ la(t0, RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr()));
+    }
+    __ jalr(t0);
+  }
+
+  // Stub exit
+  __ j(slow_continuation);
+}
+
+#undef __
+
 #endif // COMPILER2
 
 #ifdef COMPILER1
 #undef __
 #define __ ce->masm()->
 
-void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
-                                                         LIR_Opr ref) const {
-  assert_different_registers(xthread, ref->as_register(), t1);
-  __ ld(t1, address_bad_mask_from_thread(xthread));
-  __ andr(t1, t1, ref->as_register());
+static void z_color(LIR_Assembler* ce, LIR_Opr ref) {
+  __ relocate(barrier_Relocation::spec(), [&] {
+    __ li16u(t1, barrier_Relocation::unpatched);
+  }, ZBarrierRelocationFormatStoreGoodBits);
+  __ slli(ref->as_register(), ref->as_register(), ZPointerLoadShift);
+  __ orr(ref->as_register(), ref->as_register(), t1);
+}
+
+static void z_uncolor(LIR_Assembler* ce, LIR_Opr ref) {
+  __ srli(ref->as_register(), ref->as_register(), ZPointerLoadShift);
+}
+
+static void check_color(LIR_Assembler* ce, LIR_Opr ref, bool on_non_strong) {
+  assert_different_registers(t0, xthread, ref->as_register());
+  int format = on_non_strong ? ZBarrierRelocationFormatMarkBadMask
+                             : ZBarrierRelocationFormatLoadBadMask;
+  Label good;
+  __ relocate(barrier_Relocation::spec(), [&] {
+    __ li16u(t0, barrier_Relocation::unpatched);
+  }, format);
+  __ andr(t0, ref->as_register(), t0);
+}
+
+void ZBarrierSetAssembler::generate_c1_color(LIR_Assembler* ce, LIR_Opr ref) const {
+  z_color(ce, ref);
+}
+
+void ZBarrierSetAssembler::generate_c1_uncolor(LIR_Assembler* ce, LIR_Opr ref) const {
+  z_uncolor(ce, ref);
+}
+
+void ZBarrierSetAssembler::generate_c1_load_barrier(LIR_Assembler* ce,
+                                                    LIR_Opr ref,
+                                                    ZLoadBarrierStubC1* stub,
+                                                    bool on_non_strong) const {
+  Label good;
+  check_color(ce, ref, on_non_strong);
+  __ beqz(t0, good);
+  __ j(*stub->entry());
+
+  __ bind(good);
+  z_uncolor(ce, ref);
+  __ bind(*stub->continuation());
 }
 
 void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
@@ -382,42 +902,41 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
     ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
   }
 
-  assert_different_registers(ref, ref_addr, noreg);
+   assert_different_registers(ref, ref_addr, noreg);
 
-  // Save x10 unless it is the result or tmp register
-  // Set up SP to accommodate parameters and maybe x10.
-  if (ref != x10 && tmp != x10) {
-    __ sub(sp, sp, 32);
-    __ sd(x10, Address(sp, 16));
-  } else {
-    __ sub(sp, sp, 16);
-  }
+   // Save x10 unless it is the result or tmp register
+   // Set up SP to accommdate parameters and maybe x10.
+   if (ref != x10 && tmp != x10) {
+     __ sub(sp, sp, 32);
+     __ sd(x10, Address(sp, 16));
+   } else {
+     __ sub(sp, sp, 16);
+   }
 
-  // Setup arguments and call runtime stub
-  ce->store_parameter(ref_addr, 1);
-  ce->store_parameter(ref, 0);
+   // Setup arguments and call runtime stub
+   ce->store_parameter(ref_addr, 1);
+   ce->store_parameter(ref, 0);
 
-  __ far_call(stub->runtime_stub());
+   __ far_call(stub->runtime_stub());
 
-  // Verify result
-  __ verify_oop(x10);
+   // Verify result
+   __ verify_oop(x10);
 
+   // Move result into place
+   if (ref != x10) {
+     __ mv(ref, x10);
+   }
 
-  // Move result into place
-  if (ref != x10) {
-    __ mv(ref, x10);
-  }
+   // Restore x10 unless it is the result or tmp register
+   if (ref != x10 && tmp != x10) {
+     __ ld(x10, Address(sp, 16));
+     __ addi(sp, sp, 32);
+   } else {
+     __ addi(sp, sp, 16);
+   }
 
-  // Restore x10 unless it is the result or tmp register
-  if (ref != x10 && tmp != x10) {
-    __ ld(x10, Address(sp, 16));
-    __ add(sp, sp, 32);
-  } else {
-    __ add(sp, sp, 16);
-  }
-
-  // Stub exit
-  __ j(*stub->continuation());
+   // Stub exit
+   __ j(*stub->continuation());
 }
 
 #undef __
@@ -440,19 +959,131 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler*
   __ epilogue();
 }
 
+void ZBarrierSetAssembler::generate_c1_store_barrier_runtime_stub(StubAssembler* sasm,
+                                                                  bool self_healing) const {
+  __ prologue("zgc_store_barrier stub", false);
+
+  __ push_call_clobbered_registers();
+
+  // Setup arguments
+  __ load_parameter(0, c_rarg0);
+
+  if (self_healing) {
+    __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr(), 1);
+  } else {
+    __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), 1);
+  }
+
+  __ pop_call_clobbered_registers();
+
+  __ epilogue();
+}
+
+#undef __
+#define __ ce->masm()->
+
+void ZBarrierSetAssembler::generate_c1_store_barrier(LIR_Assembler* ce,
+                                                     LIR_Address* addr,
+                                                     LIR_Opr new_zaddress,
+                                                     LIR_Opr new_zpointer,
+                                                     ZStoreBarrierStubC1* stub) const {
+  Register rnew_zaddress = new_zaddress->as_register();
+  Register rnew_zpointer = new_zpointer->as_register();
+
+  store_barrier_fast(ce->masm(),
+                     ce->as_Address(addr),
+                     rnew_zaddress,
+                     rnew_zpointer,
+                     t1,
+                     true,
+                     stub->is_atomic(),
+                     *stub->entry(),
+                     *stub->continuation());
+}
+
+void ZBarrierSetAssembler::generate_c1_store_barrier_stub(LIR_Assembler* ce,
+                                                          ZStoreBarrierStubC1* stub) const {
+  // Stub entry
+  __ bind(*stub->entry());
+  Label slow;
+  Label slow_continuation;
+  store_barrier_medium(ce->masm(),
+                       ce->as_Address(stub->ref_addr()->as_address_ptr()),
+                       t1,
+                       stub->new_zpointer()->as_register(),
+                       stub->tmp()->as_pointer_register(),
+                       false /* is_native */,
+                       stub->is_atomic(),
+                       *stub->continuation(),
+                       slow,
+                       slow_continuation);
+
+  __ bind(slow);
+
+  __ la(stub->new_zpointer()->as_register(), ce->as_Address(stub->ref_addr()->as_address_ptr()));
+
+  __ sub(sp, sp, 16);
+  //Setup arguments and call runtime stub
+  assert(stub->new_zpointer()->is_valid(), "invariant");
+  ce->store_parameter(stub->new_zpointer()->as_register(), 0);
+  __ far_call(stub->runtime_stub());
+  __ addi(sp, sp, 16);
+
+  // Stub exit
+  __ j(slow_continuation);
+}
+
+#undef __
+
 #endif // COMPILER1
 
 #undef __
 #define __ masm->
 
 void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
-  // Check if mask is good.
-  // verifies that ZAddressBadMask & obj == 0
-  __ ld(tmp2, Address(xthread, ZThreadLocalData::address_bad_mask_offset()));
-  __ andr(tmp1, obj, tmp2);
-  __ bnez(tmp1, error);
+  // C1 calls verify_oop in the middle of barriers, before they have been uncolored
+  // and after being colored. Therefore, we must deal with colored oops as well.
+  Label done;
+  Label check_oop;
+  Label check_zaddress;
+  int color_bits = ZPointerRemappedShift + ZPointerRemappedBits;
 
-  BarrierSetAssembler::check_oop(masm, obj, tmp1, tmp2, error);
+  uintptr_t shifted_base_start_mask = (UCONST64(1) << (ZAddressHeapBaseShift + color_bits + 1)) - 1;
+  uintptr_t shifted_base_end_mask = (UCONST64(1) << (ZAddressHeapBaseShift + 1)) - 1;
+  uintptr_t shifted_base_mask = shifted_base_start_mask ^ shifted_base_end_mask;
+
+  uintptr_t shifted_address_end_mask = (UCONST64(1) << (color_bits + 1)) - 1;
+  uintptr_t shifted_address_mask = shifted_base_end_mask ^ (uintptr_t)CONST64(-1);
+
+  // Check colored null
+  __ mv(tmp1, shifted_address_mask);
+  __ andr(tmp1, tmp1, obj);
+  __ beqz(tmp1, done);
+
+  // Check for zpointer
+  __ mv(tmp1, shifted_base_mask);
+  __ andr(tmp1, tmp1, obj);
+  __ beqz(tmp1, check_oop);
+
+  // Uncolor presumed zpointer
+  __ srli(obj, obj, ZPointerLoadShift);
+
+  __ j(check_zaddress);
+
+  __ bind(check_oop);
+
+  // Make sure klass is 'reasonable', which is not zero
+  __ load_klass(tmp1, obj, tmp2);
+  __ beqz(tmp1, error);
+
+  __ bind(check_zaddress);
+  // Check if the oop is the right area of memory
+  __ mv(tmp1, (intptr_t) Universe::verify_oop_mask());
+  __ andr(tmp1, tmp1, obj);
+  __ mv(obj, (intptr_t) Universe::verify_oop_bits());
+  __ bne(tmp1, obj, error);
+
+  __ bind(done);
 }
 
 #undef __
diff --git a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp
index c7f29684a64..4597949d189 100644
--- a/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/gc/z/zBarrierSetAssembler_riscv.hpp
@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,22 +28,33 @@
 
 #include "code/vmreg.hpp"
 #include "oops/accessDecorators.hpp"
+#ifdef COMPILER1
+#include "c1/c1_LIR.hpp"
+#endif // COMPILER1
 #ifdef COMPILER2
+#include "gc/z/c2/zBarrierSetC2.hpp"
 #include "opto/optoreg.hpp"
 #endif // COMPILER2
 
 #ifdef COMPILER1
+class LIR_Address;
 class LIR_Assembler;
 class LIR_Opr;
 class StubAssembler;
 class ZLoadBarrierStubC1;
+class ZStoreBarrierStubC1;
 #endif // COMPILER1
 
 #ifdef COMPILER2
+class MachNode;
 class Node;
-class ZLoadBarrierStubC2;
 #endif // COMPILER2
 
+const int ZBarrierRelocationFormatLoadBadMask   = 0;
+const int ZBarrierRelocationFormatMarkBadMask   = 1;
+const int ZBarrierRelocationFormatStoreGoodBits = 2;
+const int ZBarrierRelocationFormatStoreBadMask  = 3;
+
 class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase {
 public:
   virtual void load_at(MacroAssembler* masm,
@@ -54,7 +65,27 @@ public:
                        Register tmp1,
                        Register tmp2);
 
-#ifdef ASSERT
+  void store_barrier_fast(MacroAssembler* masm,
+                          Address ref_addr,
+                          Register rnew_zaddress,
+                          Register rnew_zpointer,
+                          Register rtmp,
+                          bool in_nmethod,
+                          bool is_atomic,
+                          Label& medium_path,
+                          Label& medium_path_continuation) const;
+
+  void store_barrier_medium(MacroAssembler* masm,
+                            Address ref_addr,
+                            Register rtmp1,
+                            Register rtmp2,
+                            Register rtmp3,
+                            bool is_native,
+                            bool is_atomic,
+                            Label& medium_path_continuation,
+                            Label& slow_path,
+                            Label& slow_path_continuation) const;
+
   virtual void store_at(MacroAssembler* masm,
                         DecoratorSet decorators,
                         BasicType type,
@@ -63,7 +94,6 @@ public:
                         Register tmp1,
                         Register tmp2,
                         Register tmp3);
-#endif // ASSERT
 
   virtual void arraycopy_prologue(MacroAssembler* masm,
                                   DecoratorSet decorators,
@@ -73,23 +103,66 @@ public:
                                   Register count,
                                   RegSet saved_regs);
 
+  virtual void copy_load_at(MacroAssembler* masm,
+                            DecoratorSet decorators,
+                            BasicType type,
+                            size_t bytes,
+                            Register dst,
+                            Address src,
+                            Register tmp);
+
+  virtual void copy_store_at(MacroAssembler* masm,
+                             DecoratorSet decorators,
+                             BasicType type,
+                             size_t bytes,
+                             Address dst,
+                             Register src,
+                             Register tmp1,
+                             Register tmp2,
+                             Register tmp3);
+
+  virtual bool supports_rvv_arraycopy();
+
   virtual void try_resolve_jobject_in_native(MacroAssembler* masm,
                                              Register jni_env,
                                              Register robj,
                                              Register tmp,
                                              Label& slowpath);
 
-  virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_data_patch; }
+  virtual NMethodPatchingType nmethod_patching_type() { return NMethodPatchingType::conc_instruction_and_data_patch; }
+
+  void patch_barrier_relocation(address addr, int format);
+
+  void patch_barriers() {}
 
 #ifdef COMPILER1
+  void generate_c1_color(LIR_Assembler* ce, LIR_Opr ref) const;
+  void generate_c1_uncolor(LIR_Assembler* ce, LIR_Opr ref) const;
+
   void generate_c1_load_barrier_test(LIR_Assembler* ce,
                                      LIR_Opr ref) const;
+  void generate_c1_load_barrier(LIR_Assembler* ce,
+                                LIR_Opr ref,
+                                ZLoadBarrierStubC1* stub,
+                                bool on_non_strong) const;
 
   void generate_c1_load_barrier_stub(LIR_Assembler* ce,
                                      ZLoadBarrierStubC1* stub) const;
 
   void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
                                              DecoratorSet decorators) const;
+
+  void generate_c1_store_barrier(LIR_Assembler* ce,
+                                 LIR_Address* addr,
+                                 LIR_Opr new_zaddress,
+                                 LIR_Opr new_zpointer,
+                                 ZStoreBarrierStubC1* stub) const;
+
+  void generate_c1_store_barrier_stub(LIR_Assembler* ce,
+                                      ZStoreBarrierStubC1* stub) const;
+
+  void generate_c1_store_barrier_runtime_stub(StubAssembler* sasm,
+                                              bool self_healing) const;
 #endif // COMPILER1
 
 #ifdef COMPILER2
@@ -98,6 +171,8 @@ public:
 
   void generate_c2_load_barrier_stub(MacroAssembler* masm,
                                      ZLoadBarrierStubC2* stub) const;
+  void generate_c2_store_barrier_stub(MacroAssembler* masm,
+                                      ZStoreBarrierStubC2* stub) const;
 #endif // COMPILER2
 
   void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
diff --git a/src/hotspot/cpu/riscv/gc/z/zGlobals_riscv.hpp b/src/hotspot/cpu/riscv/gc/z/zGlobals_riscv.hpp
index bd72753b882..3f90ef07f27 100644
--- a/src/hotspot/cpu/riscv/gc/z/zGlobals_riscv.hpp
+++ b/src/hotspot/cpu/riscv/gc/z/zGlobals_riscv.hpp
@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,10 +26,8 @@
 #ifndef CPU_RISCV_GC_Z_ZGLOBALS_RISCV_HPP
 #define CPU_RISCV_GC_Z_ZGLOBALS_RISCV_HPP
 
-const size_t ZPlatformHeapViews        = 3;
+#include "utilities/globalDefinitions.hpp"
+
 const size_t ZPlatformCacheLineSize    = 64;
 
-size_t ZPlatformAddressOffsetBits();
-size_t ZPlatformAddressMetadataShift();
-
 #endif // CPU_RISCV_GC_Z_ZGLOBALS_RISCV_HPP
diff --git a/src/hotspot/cpu/riscv/gc/z/z_riscv64.ad b/src/hotspot/cpu/riscv/gc/z/z_riscv64.ad
index 6b6f87814a5..1429f4ad520 100644
--- a/src/hotspot/cpu/riscv/gc/z/z_riscv64.ad
+++ b/src/hotspot/cpu/riscv/gc/z/z_riscv64.ad
@@ -1,6 +1,6 @@
 //
-// Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
-// Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
+// Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
@@ -31,31 +31,69 @@ source_hpp %{
 %}
 
 source %{
+#include "gc/z/zBarrierSetAssembler.hpp"
 
-static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, int barrier_data) {
-  if (barrier_data == ZLoadBarrierElided) {
+static void z_color(MacroAssembler& _masm, const MachNode* node, Register dst, Register src, Register tmp) {
+  assert_different_registers(dst, tmp);
+
+  __ relocate(barrier_Relocation::spec(), [&] {
+    __ li16u(tmp, barrier_Relocation::unpatched);
+  }, ZBarrierRelocationFormatStoreGoodBits);
+  __ slli(dst, src, ZPointerLoadShift);
+  __ orr(dst, dst, tmp);
+}
+
+static void z_uncolor(MacroAssembler& _masm, const MachNode* node, Register ref) {
+  __ srli(ref, ref, ZPointerLoadShift);
+}
+
+static void check_color(MacroAssembler& _masm, Register ref, bool on_non_strong, Register result) {
+  int format = on_non_strong ? ZBarrierRelocationFormatMarkBadMask
+                             : ZBarrierRelocationFormatLoadBadMask;
+  __ relocate(barrier_Relocation::spec(), [&] {
+    __ li16u(result, barrier_Relocation::unpatched);
+  }, format);
+  __ andr(result, ref, result);
+}
+
+static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
+  const bool on_non_strong =
+      ((node->barrier_data() & ZBarrierWeak) != 0) ||
+      ((node->barrier_data() & ZBarrierPhantom) != 0);
+
+  if (node->barrier_data() == ZBarrierElided) {
+    z_uncolor(_masm, node, ref);
     return;
   }
-  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
-  __ ld(tmp, Address(xthread, ZThreadLocalData::address_bad_mask_offset()));
-  __ andr(tmp, tmp, ref);
-  __ bnez(tmp, *stub->entry(), true /* far */);
-  __ bind(*stub->continuation());
-}
 
-static void z_load_barrier_slow_path(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp) {
-  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, ZLoadBarrierStrong);
+  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref);
+  Label good;
+  check_color(_masm, ref, on_non_strong, tmp);
+  __ beqz(tmp, good);
   __ j(*stub->entry());
+
+  __ bind(good);
+  z_uncolor(_masm, node, ref);
   __ bind(*stub->continuation());
 }
 
+static void z_store_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register rnew_zaddress, Register rnew_zpointer, Register tmp, bool is_atomic) {
+  if (node->barrier_data() == ZBarrierElided) {
+    z_color(_masm, node, rnew_zpointer, rnew_zaddress, t0);
+  } else {
+    bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
+    ZStoreBarrierStubC2* const stub = ZStoreBarrierStubC2::create(node, ref_addr, rnew_zaddress, rnew_zpointer, is_native, is_atomic);
+    ZBarrierSetAssembler* bs_asm = ZBarrierSet::assembler();
+    bs_asm->store_barrier_fast(&_masm, ref_addr, rnew_zaddress, rnew_zpointer, tmp, true /* in_nmethod */, is_atomic, *stub->entry(), *stub->continuation());
+  }
+}
 %}
 
 // Load Pointer
 instruct zLoadP(iRegPNoSp dst, memory mem)
 %{
   match(Set dst (LoadP mem));
-  predicate(UseZGC && (n->as_Load()->barrier_data() != 0));
+  predicate(UseZGC && ZGenerational && n->as_Load()->barrier_data() != 0);
   effect(TEMP dst);
 
   ins_cost(4 * DEFAULT_COST);
@@ -63,19 +101,36 @@ instruct zLoadP(iRegPNoSp dst, memory mem)
   format %{ "ld  $dst, $mem, #@zLoadP" %}
 
   ins_encode %{
-    const Address ref_addr (as_Register($mem$$base), $mem$$disp);
+    const Address ref_addr(as_Register($mem$$base), $mem$$disp);
     __ ld($dst$$Register, ref_addr);
-    z_load_barrier(_masm, this, ref_addr, $dst$$Register, t0 /* tmp */, barrier_data());
+    z_load_barrier(_masm, this, ref_addr, $dst$$Register, t0);
   %}
 
   ins_pipe(iload_reg_mem);
 %}
 
-instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+// Store Pointer
+instruct zStoreP(memory mem, iRegP src, iRegPNoSp tmp, rFlagsReg cr)
+%{
+  predicate(UseZGC && ZGenerational && n->as_Store()->barrier_data() != 0);
+  match(Set mem (StoreP mem src));
+  effect(TEMP tmp, KILL cr);
+
+  ins_cost(125); // XXX
+  format %{ "sd    $mem, $src\t# ptr" %}
+  ins_encode %{
+    const Address ref_addr(as_Register($mem$$base), $mem$$disp);
+    z_store_barrier(_masm, this, ref_addr, $src$$Register, $tmp$$Register, t1, false /* is_atomic */);
+    __ sd($tmp$$Register, ref_addr);
+  %}
+  ins_pipe(pipe_serial);
+%}
+
+instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
-  predicate(UseZGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
-  effect(KILL cr, TEMP_DEF res);
+  predicate(UseZGC && ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, KILL cr, TEMP_DEF res);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
@@ -83,35 +138,21 @@ instruct zCompareAndSwapP(iRegINoSp res, indirect mem, iRegP oldval, iRegP newva
             "mv $res, $res == $oldval" %}
 
   ins_encode %{
-    Label failed;
-    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
-    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
-               Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register,
-               true /* result_as_bool */);
-    __ beqz($res$$Register, failed);
-    __ mv(t0, $oldval$$Register);
-    __ bind(failed);
-    if (barrier_data() != ZLoadBarrierElided) {
-      Label good;
-      __ ld(t1, Address(xthread, ZThreadLocalData::address_bad_mask_offset()), t1 /* tmp */);
-      __ andr(t1, t1, t0);
-      __ beqz(t1, good);
-      z_load_barrier_slow_path(_masm, this, Address($mem$$Register), t0 /* ref */, t1 /* tmp */);
-      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
-                 Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register,
-                 true /* result_as_bool */);
-      __ bind(good);
-    }
+    guarantee($mem$$disp == 0, "impossible encoding");
+    Address ref_addr($mem$$Register);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, t0);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, t1, true /* is_atomic */);
+    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register, true /* result_as_bool */);
   %}
 
   ins_pipe(pipe_slow);
 %}
 
-instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, rFlagsReg cr) %{
+instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
-  predicate(UseZGC && needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong));
-  effect(KILL cr, TEMP_DEF res);
+  predicate(UseZGC && ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, KILL cr, TEMP_DEF res);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
@@ -119,81 +160,53 @@ instruct zCompareAndSwapPAcq(iRegINoSp res, indirect mem, iRegP oldval, iRegP ne
             "mv $res, $res == $oldval" %}
 
   ins_encode %{
-    Label failed;
-    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
-    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
-               Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register,
-               true /* result_as_bool */);
-    __ beqz($res$$Register, failed);
-    __ mv(t0, $oldval$$Register);
-    __ bind(failed);
-    if (barrier_data() != ZLoadBarrierElided) {
-      Label good;
-      __ ld(t1, Address(xthread, ZThreadLocalData::address_bad_mask_offset()), t1 /* tmp */);
-      __ andr(t1, t1, t0);
-      __ beqz(t1, good);
-      z_load_barrier_slow_path(_masm, this, Address($mem$$Register), t0 /* ref */, t1 /* tmp */);
-      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
-                 Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register,
-                 true /* result_as_bool */);
-      __ bind(good);
-    }
+    guarantee($mem$$disp == 0, "impossible encoding");
+    Address ref_addr($mem$$Register);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, t0);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, t1, true /* is_atomic */);
+    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register, true /* result_as_bool */);
   %}
 
   ins_pipe(pipe_slow);
 %}
 
-instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval) %{
+instruct zCompareAndExchangeP(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
-  predicate(UseZGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
-  effect(TEMP_DEF res);
+  predicate(UseZGC && ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, KILL cr, TEMP_DEF res);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
   format %{ "cmpxchg $res = $mem, $oldval, $newval, #@zCompareAndExchangeP" %}
 
   ins_encode %{
-    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
-    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
-               Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register);
-    if (barrier_data() != ZLoadBarrierElided) {
-      Label good;
-      __ ld(t0, Address(xthread, ZThreadLocalData::address_bad_mask_offset()));
-      __ andr(t0, t0, $res$$Register);
-      __ beqz(t0, good);
-      z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, t0 /* tmp */);
-      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
-                 Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register);
-      __ bind(good);
-    }
+    guarantee($mem$$disp == 0, "impossible encoding");
+    Address ref_addr($mem$$Register);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, t0);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, t1, true /* is_atomic */);
+    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::relaxed /* acquire */, Assembler::rl /* release */, $res$$Register);
+    z_uncolor(_masm, this, $res$$Register);
   %}
 
   ins_pipe(pipe_slow);
 %}
 
-instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval) %{
+instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iRegP newval, iRegPNoSp oldval_tmp, iRegPNoSp newval_tmp, rFlagsReg cr) %{
   match(Set res (CompareAndExchangeP mem (Binary oldval newval)));
-  predicate(UseZGC && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
-  effect(TEMP_DEF res);
+  predicate(UseZGC && ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP oldval_tmp, TEMP newval_tmp, KILL cr, TEMP_DEF res);
 
   ins_cost(2 * VOLATILE_REF_COST);
 
   format %{ "cmpxchg $res = $mem, $oldval, $newval, #@zCompareAndExchangePAcq" %}
 
   ins_encode %{
-    guarantee($mem$$index == -1 && $mem$$disp == 0, "impossible encoding");
-    __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
-               Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register);
-    if (barrier_data() != ZLoadBarrierElided) {
-      Label good;
-      __ ld(t0, Address(xthread, ZThreadLocalData::address_bad_mask_offset()));
-      __ andr(t0, t0, $res$$Register);
-      __ beqz(t0, good);
-      z_load_barrier_slow_path(_masm, this, Address($mem$$Register), $res$$Register /* ref */, t0 /* tmp */);
-      __ cmpxchg($mem$$Register, $oldval$$Register, $newval$$Register, Assembler::int64,
-                 Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register);
-      __ bind(good);
-    }
+    guarantee($mem$$disp == 0, "impossible encoding");
+    Address ref_addr($mem$$Register);
+    z_color(_masm, this, $oldval_tmp$$Register, $oldval$$Register, t0);
+    z_store_barrier(_masm, this, ref_addr, $newval$$Register, $newval_tmp$$Register, t1, true /* is_atomic */);
+    __ cmpxchg($mem$$Register, $oldval_tmp$$Register, $newval_tmp$$Register, Assembler::int64, Assembler::aq /* acquire */, Assembler::rl /* release */, $res$$Register);
+    z_uncolor(_masm, this, $res$$Register);
   %}
 
   ins_pipe(pipe_slow);
@@ -201,7 +214,7 @@ instruct zCompareAndExchangePAcq(iRegPNoSp res, indirect mem, iRegP oldval, iReg
 
 instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
   match(Set prev (GetAndSetP mem newv));
-  predicate(UseZGC && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
+  predicate(UseZGC && ZGenerational && !needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
   effect(TEMP_DEF prev, KILL cr);
 
   ins_cost(2 * VOLATILE_REF_COST);
@@ -209,8 +222,9 @@ instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
   format %{ "atomic_xchg  $prev, $newv, [$mem], #@zGetAndSetP" %}
 
   ins_encode %{
-    __ atomic_xchg($prev$$Register, $newv$$Register, as_Register($mem$$base));
-    z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, t0 /* tmp */, barrier_data());
+    z_store_barrier(_masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, t1, true /* is_atomic */);
+    __ atomic_xchg($prev$$Register, $prev$$Register, $mem$$Register);
+    z_uncolor(_masm, this, $prev$$Register);
   %}
 
   ins_pipe(pipe_serial);
@@ -218,16 +232,17 @@ instruct zGetAndSetP(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
 
 instruct zGetAndSetPAcq(indirect mem, iRegP newv, iRegPNoSp prev, rFlagsReg cr) %{
   match(Set prev (GetAndSetP mem newv));
-  predicate(UseZGC && needs_acquiring_load_reserved(n) && (n->as_LoadStore()->barrier_data() != 0));
+  predicate(UseZGC && ZGenerational && needs_acquiring_load_reserved(n) && n->as_LoadStore()->barrier_data() != 0);
   effect(TEMP_DEF prev, KILL cr);
 
-  ins_cost(VOLATILE_REF_COST);
+  ins_cost(2 * VOLATILE_REF_COST);
 
   format %{ "atomic_xchg_acq  $prev, $newv, [$mem], #@zGetAndSetPAcq" %}
 
   ins_encode %{
-    __ atomic_xchgal($prev$$Register, $newv$$Register, as_Register($mem$$base));
-    z_load_barrier(_masm, this, Address(noreg, 0), $prev$$Register, t0 /* tmp */, barrier_data());
+    z_store_barrier(_masm, this, Address($mem$$Register), $newv$$Register, $prev$$Register, t1, true /* is_atomic */);
+    __ atomic_xchgal($prev$$Register, $prev$$Register, $mem$$Register);
+    z_uncolor(_masm, this, $prev$$Register);
   %}
   ins_pipe(pipe_serial);
 %}
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
index c47cc1c9677..b04aebf80dd 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
@@ -563,8 +563,8 @@ void MacroAssembler::resolve_jobject(Register value, Register tmp1, Register tmp
 
   beqz(value, done);           // Use null as-is.
   // Test for tag.
-  andi(t0, value, JNIHandles::tag_mask);
-  bnez(t0, tagged);
+  andi(tmp1, value, JNIHandles::tag_mask);
+  bnez(tmp1, tagged);
 
   // Resolve local handle
   access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp1, tmp2);
@@ -573,12 +573,14 @@ void MacroAssembler::resolve_jobject(Register value, Register tmp1, Register tmp
 
   bind(tagged);
   // Test for jweak tag.
-  test_bit(t0, value, exact_log2(JNIHandles::TypeTag::weak_global));
-  bnez(t0, weak_tagged);
+  STATIC_ASSERT(JNIHandles::TypeTag::weak_global == 0b1);
+  test_bit(tmp1, value, exact_log2(JNIHandles::TypeTag::weak_global));
+  bnez(tmp1, weak_tagged);
 
   // Resolve global handle
   access_load_at(T_OBJECT, IN_NATIVE, value,
                  Address(value, -JNIHandles::TypeTag::global), tmp1, tmp2);
+  verify_oop(value);
   j(done);
 
   bind(weak_tagged);
@@ -598,9 +600,10 @@ void MacroAssembler::resolve_global_jobject(Register value, Register tmp1, Regis
 
 #ifdef ASSERT
   {
+    STATIC_ASSERT(JNIHandles::TypeTag::global == 0b10);
     Label valid_global_tag;
-    test_bit(t0, value, exact_log2(JNIHandles::TypeTag::global)); // Test for global tag.
-    bnez(t0, valid_global_tag);
+    test_bit(tmp1, value, exact_log2(JNIHandles::TypeTag::global)); // Test for global tag.
+    bnez(tmp1, valid_global_tag);
     stop("non global jobject using resolve_global_jobject");
     bind(valid_global_tag);
   }
@@ -755,6 +758,11 @@ void MacroAssembler::la(Register Rd, Label &label) {
   wrap_label(Rd, label, &MacroAssembler::la);
 }
 
+void MacroAssembler::li16u(Register Rd, int32_t imm) {
+  lui(Rd, imm << 12);
+  srli(Rd, Rd, 12);
+}
+
 void MacroAssembler::li32(Register Rd, int32_t imm) {
   // int32_t is in range 0x8000 0000 ~ 0x7fff ffff, and imm[31] is the sign bit
   int64_t upper = imm, lower = imm;
@@ -1404,6 +1412,11 @@ static int patch_imm_in_li64(address branch, address target) {
   return LI64_INSTRUCTIONS_NUM * NativeInstruction::instruction_size;
 }
 
+static int patch_imm_in_li16u(address branch, int32_t target) {
+  Assembler::patch(branch, 31, 12, target & 0xfffff); // patch lui only
+  return NativeInstruction::instruction_size;
+}
+
 int MacroAssembler::patch_imm_in_li32(address branch, int32_t target) {
   const int LI32_INSTRUCTIONS_NUM = 2;                                          // lui + addiw
   int64_t upper = (intptr_t)target;
@@ -1493,6 +1506,9 @@ int MacroAssembler::pd_patch_instruction_size(address branch, address target) {
   } else if (NativeInstruction::is_li32_at(branch)) {                 // li32
     int64_t imm = (intptr_t)target;
     return patch_imm_in_li32(branch, (int32_t)imm);
+  } else if (NativeInstruction::is_li16u_at(branch)) {
+    int64_t imm = (intptr_t)target;
+    return patch_imm_in_li16u(branch, (int32_t)imm);
   } else {
 #ifdef ASSERT
     tty->print_cr("pd_patch_instruction_size: instruction 0x%x at " INTPTR_FORMAT " could not be patched!\n",
@@ -2426,6 +2442,10 @@ void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acqui
 
 void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp,
                                 Label &succeed, Label *fail) {
+  assert_different_registers(addr, tmp);
+  assert_different_registers(newv, tmp);
+  assert_different_registers(oldv, tmp);
+
   // oldv holds comparison value
   // newv holds value to write in exchange
   // addr identifies memory word to compare against/update
@@ -2612,6 +2632,9 @@ void MacroAssembler::cmpxchg(Register addr, Register expected,
                              Assembler::Aqrl acquire, Assembler::Aqrl release,
                              Register result, bool result_as_bool) {
   assert(size != int8 && size != int16, "unsupported operand size");
+  assert_different_registers(addr, t0);
+  assert_different_registers(expected, t0);
+  assert_different_registers(new_val, t0);
 
   Label retry_load, done, ne_done;
   bind(retry_load);
@@ -2644,6 +2667,10 @@ void MacroAssembler::cmpxchg_weak(Register addr, Register expected,
                                   enum operand_size size,
                                   Assembler::Aqrl acquire, Assembler::Aqrl release,
                                   Register result) {
+  assert_different_registers(addr, t0);
+  assert_different_registers(expected, t0);
+  assert_different_registers(new_val, t0);
+
   Label fail, done;
   load_reserved(addr, size, acquire);
   bne(t0, expected, fail);
diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
index e6a286d4ea3..143e71413f1 100644
--- a/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
+++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
@@ -689,6 +689,7 @@ public:
   void la(Register Rd, const address dest);
   void la(Register Rd, const Address &adr);
 
+  void li16u(Register Rd, int32_t imm);
   void li32(Register Rd, int32_t imm);
   void li64(Register Rd, int64_t imm);
   void li  (Register Rd, int64_t imm);  // optimized load immediate
diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
index a0a946fb5fe..ee91ceae809 100644
--- a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
+++ b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp
@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
- * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -97,6 +97,12 @@ bool NativeInstruction::is_movptr_at(address instr) {
          check_movptr_data_dependency(instr);
 }
 
+bool NativeInstruction::is_li16u_at(address instr) {
+  return is_lui_at(instr) && // lui
+         is_srli_at(instr + instruction_size) && // srli
+         check_li16u_data_dependency(instr);
+}
+
 bool NativeInstruction::is_li32_at(address instr) {
   return is_lui_at(instr) && // lui
          is_addiw_at(instr + instruction_size) && // addiw
diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp
index 8534b873d9a..df69b33c364 100644
--- a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp
+++ b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp
@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2018, Red Hat Inc. All rights reserved.
- * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -81,6 +81,14 @@ class NativeInstruction {
   static bool is_addiw_to_zr_at(address instr) { assert_cond(instr != nullptr); return is_addiw_at(instr) && extract_rd(instr) == zr; }
   static bool is_lui_at(address instr)        { assert_cond(instr != nullptr); return extract_opcode(instr) == 0b0110111; }
   static bool is_lui_to_zr_at(address instr)  { assert_cond(instr != nullptr); return is_lui_at(instr) && extract_rd(instr) == zr; }
+
+  static bool is_srli_at(address instr) {
+    assert_cond(instr != nullptr);
+    return extract_opcode(instr) == 0b0010011 &&
+           extract_funct3(instr) == 0b101 &&
+           Assembler::extract(((unsigned*)instr)[0], 31, 26) == 0b000000;
+  }
+
   static bool is_slli_shift_at(address instr, uint32_t shift) {
     assert_cond(instr != nullptr);
     return (extract_opcode(instr) == 0b0010011 && // opcode field
@@ -153,6 +161,17 @@ class NativeInstruction {
            extract_rs1(addi4) == extract_rd(addi4);
   }
 
+  // the instruction sequence of li16u is as below:
+  //     lui
+  //     srli
+  static bool check_li16u_data_dependency(address instr) {
+    address lui = instr;
+    address srli = lui + instruction_size;
+
+    return extract_rs1(srli) == extract_rd(lui) &&
+           extract_rs1(srli) == extract_rd(srli);
+  }
+
   // the instruction sequence of li32 is as below:
   //     lui
   //     addiw
@@ -186,6 +205,7 @@ class NativeInstruction {
   }
 
   static bool is_movptr_at(address instr);
+  static bool is_li16u_at(address instr);
   static bool is_li32_at(address instr);
   static bool is_li64_at(address instr);
   static bool is_pc_relative_at(address branch);
diff --git a/src/hotspot/cpu/riscv/relocInfo_riscv.hpp b/src/hotspot/cpu/riscv/relocInfo_riscv.hpp
index 840ed935d88..8f247884944 100644
--- a/src/hotspot/cpu/riscv/relocInfo_riscv.hpp
+++ b/src/hotspot/cpu/riscv/relocInfo_riscv.hpp
@@ -1,6 +1,6 @@
 /*
- * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
- * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,7 +32,8 @@
     // Relocations are byte-aligned.
     offset_unit        =  1,
     // Must be at least 1 for RelocInfo::narrow_oop_in_const.
-    format_width       =  1
+    // Must be at least 2 for ZGC GC barrier patching.
+    format_width       =  2
   };
 
  public:
diff --git a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
index d1c76beb3f9..ab6c3e8feb0 100644
--- a/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
+++ b/src/hotspot/cpu/riscv/stubGenerator_riscv.cpp
@@ -51,9 +51,6 @@
 #ifdef COMPILER2
 #include "opto/runtime.hpp"
 #endif
-#if INCLUDE_ZGC
-#include "gc/z/zThreadLocalData.hpp"
-#endif
 
 // Declaration and definition of StubGenerator (no .hpp file).
 // For a more detailed description of the stub routine structure
@@ -957,7 +954,11 @@ class StubGenerator: public StubCodeGenerator {
     Label same_aligned;
     Label copy_big, copy32_loop, copy8_loop, copy_small, done;
 
-    __ beqz(count, done);
+    // The size of copy32_loop body increases significantly with ZGC GC barriers.
+    // Need conditional far branches to reach a point beyond the loop in this case.
+    bool is_far = UseZGC && ZGenerational;
+
+    __ beqz(count, done, is_far);
     __ slli(cnt, count, exact_log2(granularity));
     if (is_backwards) {
       __ add(src, s, cnt);
@@ -971,15 +972,15 @@ class StubGenerator: public StubCodeGenerator {
       __ addi(t0, cnt, -32);
       __ bgez(t0, copy32_loop);
       __ addi(t0, cnt, -8);
-      __ bgez(t0, copy8_loop);
+      __ bgez(t0, copy8_loop, is_far);
       __ j(copy_small);
     } else {
       __ mv(t0, 16);
-      __ blt(cnt, t0, copy_small);
+      __ blt(cnt, t0, copy_small, is_far);
 
       __ xorr(t0, src, dst);
       __ andi(t0, t0, 0b111);
-      __ bnez(t0, copy_small);
+      __ bnez(t0, copy_small, is_far);
 
       __ bind(same_aligned);
       __ andi(t0, src, 0b111);
@@ -995,26 +996,27 @@ class StubGenerator: public StubCodeGenerator {
         __ addi(dst, dst, step);
       }
       __ addi(cnt, cnt, -granularity);
-      __ beqz(cnt, done);
+      __ beqz(cnt, done, is_far);
       __ j(same_aligned);
 
       __ bind(copy_big);
       __ mv(t0, 32);
-      __ blt(cnt, t0, copy8_loop);
+      __ blt(cnt, t0, copy8_loop, is_far);
     }
+
     __ bind(copy32_loop);
     if (is_backwards) {
       __ addi(src, src, -wordSize * 4);
       __ addi(dst, dst, -wordSize * 4);
     }
     // we first load 32 bytes, then write it, so the direction here doesn't matter
-    bs_asm->copy_load_at(_masm, decorators, type, 8, tmp3, Address(src), gct1);
-    bs_asm->copy_load_at(_masm, decorators, type, 8, tmp4, Address(src, 8), gct1);
+    bs_asm->copy_load_at(_masm, decorators, type, 8, tmp3, Address(src),     gct1);
+    bs_asm->copy_load_at(_masm, decorators, type, 8, tmp4, Address(src, 8),  gct1);
     bs_asm->copy_load_at(_masm, decorators, type, 8, tmp5, Address(src, 16), gct1);
     bs_asm->copy_load_at(_masm, decorators, type, 8, tmp6, Address(src, 24), gct1);
 
-    bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst), tmp3, gct1, gct2, gct3);
-    bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst, 8), tmp4, gct1, gct2, gct3);
+    bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst),     tmp3, gct1, gct2, gct3);
+    bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst, 8),  tmp4, gct1, gct2, gct3);
     bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst, 16), tmp5, gct1, gct2, gct3);
     bs_asm->copy_store_at(_masm, decorators, type, 8, Address(dst, 24), tmp6, gct1, gct2, gct3);
 
@@ -3731,7 +3733,7 @@ class StubGenerator: public StubCodeGenerator {
       framesize // inclusive of return address
     };
 
-    const int insts_size = 512;
+    const int insts_size = 1024;
     const int locs_size  = 64;
 
     CodeBuffer code(name, insts_size, locs_size);
@@ -3970,7 +3972,7 @@ class StubGenerator: public StubCodeGenerator {
       framesize // inclusive of return address
     };
 
-    int insts_size = 512;
+    int insts_size = 1024;
     int locs_size = 64;
     CodeBuffer code("jfr_write_checkpoint", insts_size, locs_size);
     OopMapSet* oop_maps = new OopMapSet();
diff --git a/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp b/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp
index cd2dc90c765..1a177a0c1ad 100644
--- a/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp
+++ b/src/hotspot/cpu/riscv/stubRoutines_riscv.hpp
@@ -1,7 +1,7 @@
 /*
  * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
- * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved.
+ * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -39,8 +39,8 @@ enum platform_dependent_constants {
   // simply increase sizes if too small (assembler will crash if too small)
   _initial_stubs_code_size      = 19000,
   _continuation_stubs_code_size =  2000,
-  _compiler_stubs_code_size     = 28000,
-  _final_stubs_code_size        = 28000
+  _compiler_stubs_code_size     = 128000,
+  _final_stubs_code_size        = 128000
 };
 
 class riscv {
diff --git a/src/hotspot/cpu/x86/assembler_x86.hpp b/src/hotspot/cpu/x86/assembler_x86.hpp
index c45a1580cb9..d551442ecf2 100644
--- a/src/hotspot/cpu/x86/assembler_x86.hpp
+++ b/src/hotspot/cpu/x86/assembler_x86.hpp
@@ -822,6 +822,7 @@ private:
 
   // These are all easily abused and hence protected
 
+ public:
   // 32BIT ONLY SECTION
 #ifndef _LP64
   // Make these disappear in 64bit mode since they would never be correct
@@ -843,6 +844,7 @@ private:
   void mov_narrow_oop(Address dst, int32_t imm32, RelocationHolder const& rspec);
 #endif // _LP64
 
+ protected:
   // These are unique in that we are ensured by the caller that the 32bit
   // relative in these instructions will always be able to reach the potentially
   // 64bit address described by entry. Since they can take a 64bit address they
diff --git a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
index 88b45516fdb..db70f77c8c2 100644
--- a/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp
@@ -1350,8 +1350,8 @@ void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_Patch
     }
 #endif
 
-    // Load barrier has not yet been applied, so ZGC can't verify the oop here
-    if (!UseZGC) {
+    if (!(UseZGC && !ZGenerational)) {
+      // Load barrier has not yet been applied, so ZGC can't verify the oop here
       __ verify_oop(dest->as_register());
     }
   }
diff --git a/src/hotspot/cpu/x86/gc/x/xBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/x/xBarrierSetAssembler_x86.cpp
new file mode 100644
index 00000000000..38129a9fc81
--- /dev/null
+++ b/src/hotspot/cpu/x86/gc/x/xBarrierSetAssembler_x86.cpp
@@ -0,0 +1,713 @@
+/*
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "asm/macroAssembler.inline.hpp"
+#include "code/codeBlob.hpp"
+#include "code/vmreg.inline.hpp"
+#include "gc/x/xBarrier.inline.hpp"
+#include "gc/x/xBarrierSet.hpp"
+#include "gc/x/xBarrierSetAssembler.hpp"
+#include "gc/x/xBarrierSetRuntime.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+#include "memory/resourceArea.hpp"
+#include "runtime/sharedRuntime.hpp"
+#include "utilities/macros.hpp"
+#ifdef COMPILER1
+#include "c1/c1_LIRAssembler.hpp"
+#include "c1/c1_MacroAssembler.hpp"
+#include "gc/x/c1/xBarrierSetC1.hpp"
+#endif // COMPILER1
+#ifdef COMPILER2
+#include "gc/x/c2/xBarrierSetC2.hpp"
+#endif // COMPILER2
+
+#ifdef PRODUCT
+#define BLOCK_COMMENT(str) /* nothing */
+#else
+#define BLOCK_COMMENT(str) __ block_comment(str)
+#endif
+
+#undef __
+#define __ masm->
+
+static void call_vm(MacroAssembler* masm,
+                    address entry_point,
+                    Register arg0,
+                    Register arg1) {
+  // Setup arguments
+  if (arg1 == c_rarg0) {
+    if (arg0 == c_rarg1) {
+      __ xchgptr(c_rarg1, c_rarg0);
+    } else {
+      __ movptr(c_rarg1, arg1);
+      __ movptr(c_rarg0, arg0);
+    }
+  } else {
+    if (arg0 != c_rarg0) {
+      __ movptr(c_rarg0, arg0);
+    }
+    if (arg1 != c_rarg1) {
+      __ movptr(c_rarg1, arg1);
+    }
+  }
+
+  // Call VM
+  __ MacroAssembler::call_VM_leaf_base(entry_point, 2);
+}
+
+void XBarrierSetAssembler::load_at(MacroAssembler* masm,
+                                   DecoratorSet decorators,
+                                   BasicType type,
+                                   Register dst,
+                                   Address src,
+                                   Register tmp1,
+                                   Register tmp_thread) {
+  if (!XBarrierSet::barrier_needed(decorators, type)) {
+    // Barrier not needed
+    BarrierSetAssembler::load_at(masm, decorators, type, dst, src, tmp1, tmp_thread);
+    return;
+  }
+
+  BLOCK_COMMENT("XBarrierSetAssembler::load_at {");
+
+  // Allocate scratch register
+  Register scratch = tmp1;
+  if (tmp1 == noreg) {
+    scratch = r12;
+    __ push(scratch);
+  }
+
+  assert_different_registers(dst, scratch);
+
+  Label done;
+
+  //
+  // Fast Path
+  //
+
+  // Load address
+  __ lea(scratch, src);
+
+  // Load oop at address
+  __ movptr(dst, Address(scratch, 0));
+
+  // Test address bad mask
+  __ testptr(dst, address_bad_mask_from_thread(r15_thread));
+  __ jcc(Assembler::zero, done);
+
+  //
+  // Slow path
+  //
+
+  // Save registers
+  __ push(rax);
+  __ push(rcx);
+  __ push(rdx);
+  __ push(rdi);
+  __ push(rsi);
+  __ push(r8);
+  __ push(r9);
+  __ push(r10);
+  __ push(r11);
+
+  // We may end up here from generate_native_wrapper, then the method may have
+  // floats as arguments, and we must spill them before calling the VM runtime
+  // leaf. From the interpreter all floats are passed on the stack.
+  assert(Argument::n_float_register_parameters_j == 8, "Assumption");
+  const int xmm_size = wordSize * 2;
+  const int xmm_spill_size = xmm_size * Argument::n_float_register_parameters_j;
+  __ subptr(rsp, xmm_spill_size);
+  __ movdqu(Address(rsp, xmm_size * 7), xmm7);
+  __ movdqu(Address(rsp, xmm_size * 6), xmm6);
+  __ movdqu(Address(rsp, xmm_size * 5), xmm5);
+  __ movdqu(Address(rsp, xmm_size * 4), xmm4);
+  __ movdqu(Address(rsp, xmm_size * 3), xmm3);
+  __ movdqu(Address(rsp, xmm_size * 2), xmm2);
+  __ movdqu(Address(rsp, xmm_size * 1), xmm1);
+  __ movdqu(Address(rsp, xmm_size * 0), xmm0);
+
+  // Call VM
+  call_vm(masm, XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), dst, scratch);
+
+  __ movdqu(xmm0, Address(rsp, xmm_size * 0));
+  __ movdqu(xmm1, Address(rsp, xmm_size * 1));
+  __ movdqu(xmm2, Address(rsp, xmm_size * 2));
+  __ movdqu(xmm3, Address(rsp, xmm_size * 3));
+  __ movdqu(xmm4, Address(rsp, xmm_size * 4));
+  __ movdqu(xmm5, Address(rsp, xmm_size * 5));
+  __ movdqu(xmm6, Address(rsp, xmm_size * 6));
+  __ movdqu(xmm7, Address(rsp, xmm_size * 7));
+  __ addptr(rsp, xmm_spill_size);
+
+  __ pop(r11);
+  __ pop(r10);
+  __ pop(r9);
+  __ pop(r8);
+  __ pop(rsi);
+  __ pop(rdi);
+  __ pop(rdx);
+  __ pop(rcx);
+
+  if (dst == rax) {
+    __ addptr(rsp, wordSize);
+  } else {
+    __ movptr(dst, rax);
+    __ pop(rax);
+  }
+
+  __ bind(done);
+
+  // Restore scratch register
+  if (tmp1 == noreg) {
+    __ pop(scratch);
+  }
+
+  BLOCK_COMMENT("} XBarrierSetAssembler::load_at");
+}
+
+#ifdef ASSERT
+
+void XBarrierSetAssembler::store_at(MacroAssembler* masm,
+                                    DecoratorSet decorators,
+                                    BasicType type,
+                                    Address dst,
+                                    Register src,
+                                    Register tmp1,
+                                    Register tmp2,
+                                    Register tmp3) {
+  BLOCK_COMMENT("XBarrierSetAssembler::store_at {");
+
+  // Verify oop store
+  if (is_reference_type(type)) {
+    // Note that src could be noreg, which means we
+    // are storing null and can skip verification.
+    if (src != noreg) {
+      Label done;
+      __ testptr(src, address_bad_mask_from_thread(r15_thread));
+      __ jcc(Assembler::zero, done);
+      __ stop("Verify oop store failed");
+      __ should_not_reach_here();
+      __ bind(done);
+    }
+  }
+
+  // Store value
+  BarrierSetAssembler::store_at(masm, decorators, type, dst, src, tmp1, tmp2, tmp3);
+
+  BLOCK_COMMENT("} XBarrierSetAssembler::store_at");
+}
+
+#endif // ASSERT
+
+void XBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
+                                              DecoratorSet decorators,
+                                              BasicType type,
+                                              Register src,
+                                              Register dst,
+                                              Register count) {
+  if (!XBarrierSet::barrier_needed(decorators, type)) {
+    // Barrier not needed
+    return;
+  }
+
+  BLOCK_COMMENT("XBarrierSetAssembler::arraycopy_prologue {");
+
+  // Save registers
+  __ pusha();
+
+  // Call VM
+  call_vm(masm, XBarrierSetRuntime::load_barrier_on_oop_array_addr(), src, count);
+
+  // Restore registers
+  __ popa();
+
+  BLOCK_COMMENT("} XBarrierSetAssembler::arraycopy_prologue");
+}
+
+void XBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
+                                                         Register jni_env,
+                                                         Register obj,
+                                                         Register tmp,
+                                                         Label& slowpath) {
+  BLOCK_COMMENT("XBarrierSetAssembler::try_resolve_jobject_in_native {");
+
+  // Resolve jobject
+  BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
+
+  // Test address bad mask
+  __ testptr(obj, address_bad_mask_from_jni_env(jni_env));
+  __ jcc(Assembler::notZero, slowpath);
+
+  BLOCK_COMMENT("} XBarrierSetAssembler::try_resolve_jobject_in_native");
+}
+
+#ifdef COMPILER1
+
+#undef __
+#define __ ce->masm()->
+
+void XBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
+                                                         LIR_Opr ref) const {
+  __ testptr(ref->as_register(), address_bad_mask_from_thread(r15_thread));
+}
+
+void XBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
+                                                         XLoadBarrierStubC1* stub) const {
+  // Stub entry
+  __ bind(*stub->entry());
+
+  Register ref = stub->ref()->as_register();
+  Register ref_addr = noreg;
+  Register tmp = noreg;
+
+  if (stub->tmp()->is_valid()) {
+    // Load address into tmp register
+    ce->leal(stub->ref_addr(), stub->tmp());
+    ref_addr = tmp = stub->tmp()->as_pointer_register();
+  } else {
+    // Address already in register
+    ref_addr = stub->ref_addr()->as_address_ptr()->base()->as_pointer_register();
+  }
+
+  assert_different_registers(ref, ref_addr, noreg);
+
+  // Save rax unless it is the result or tmp register
+  if (ref != rax && tmp != rax) {
+    __ push(rax);
+  }
+
+  // Setup arguments and call runtime stub
+  __ subptr(rsp, 2 * BytesPerWord);
+  ce->store_parameter(ref_addr, 1);
+  ce->store_parameter(ref, 0);
+  __ call(RuntimeAddress(stub->runtime_stub()));
+  __ addptr(rsp, 2 * BytesPerWord);
+
+  // Verify result
+  __ verify_oop(rax);
+
+  // Move result into place
+  if (ref != rax) {
+    __ movptr(ref, rax);
+  }
+
+  // Restore rax unless it is the result or tmp register
+  if (ref != rax && tmp != rax) {
+    __ pop(rax);
+  }
+
+  // Stub exit
+  __ jmp(*stub->continuation());
+}
+
+#undef __
+#define __ sasm->
+
+void XBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
+                                                                 DecoratorSet decorators) const {
+  // Enter and save registers
+  __ enter();
+  __ save_live_registers_no_oop_map(true /* save_fpu_registers */);
+
+  // Setup arguments
+  __ load_parameter(1, c_rarg1);
+  __ load_parameter(0, c_rarg0);
+
+  // Call VM
+  __ call_VM_leaf(XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), c_rarg0, c_rarg1);
+
+  // Restore registers and return
+  __ restore_live_registers_except_rax(true /* restore_fpu_registers */);
+  __ leave();
+  __ ret(0);
+}
+
+#endif // COMPILER1
+
+#ifdef COMPILER2
+
+OptoReg::Name XBarrierSetAssembler::refine_register(const Node* node, OptoReg::Name opto_reg) {
+  if (!OptoReg::is_reg(opto_reg)) {
+    return OptoReg::Bad;
+  }
+
+  const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+  if (vm_reg->is_XMMRegister()) {
+    opto_reg &= ~15;
+    switch (node->ideal_reg()) {
+      case Op_VecX:
+        opto_reg |= 2;
+        break;
+      case Op_VecY:
+        opto_reg |= 4;
+        break;
+      case Op_VecZ:
+        opto_reg |= 8;
+        break;
+      default:
+        opto_reg |= 1;
+        break;
+    }
+  }
+
+  return opto_reg;
+}
+
+// We use the vec_spill_helper from the x86.ad file to avoid reinventing this wheel
+extern void vec_spill_helper(CodeBuffer *cbuf, bool is_load,
+                            int stack_offset, int reg, uint ireg, outputStream* st);
+
+#undef __
+#define __ _masm->
+
+class XSaveLiveRegisters {
+private:
+  struct XMMRegisterData {
+    XMMRegister _reg;
+    int         _size;
+
+    // Used by GrowableArray::find()
+    bool operator == (const XMMRegisterData& other) {
+      return _reg == other._reg;
+    }
+  };
+
+  MacroAssembler* const          _masm;
+  GrowableArray<Register>        _gp_registers;
+  GrowableArray<KRegister>       _opmask_registers;
+  GrowableArray<XMMRegisterData> _xmm_registers;
+  int                            _spill_size;
+  int                            _spill_offset;
+
+  static int xmm_compare_register_size(XMMRegisterData* left, XMMRegisterData* right) {
+    if (left->_size == right->_size) {
+      return 0;
+    }
+
+    return (left->_size < right->_size) ? -1 : 1;
+  }
+
+  static int xmm_slot_size(OptoReg::Name opto_reg) {
+    // The low order 4 bytes denote what size of the XMM register is live
+    return (opto_reg & 15) << 3;
+  }
+
+  static uint xmm_ideal_reg_for_size(int reg_size) {
+    switch (reg_size) {
+    case 8:
+      return Op_VecD;
+    case 16:
+      return Op_VecX;
+    case 32:
+      return Op_VecY;
+    case 64:
+      return Op_VecZ;
+    default:
+      fatal("Invalid register size %d", reg_size);
+      return 0;
+    }
+  }
+
+  bool xmm_needs_vzeroupper() const {
+    return _xmm_registers.is_nonempty() && _xmm_registers.at(0)._size > 16;
+  }
+
+  void xmm_register_save(const XMMRegisterData& reg_data) {
+    const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg());
+    const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size);
+    _spill_offset -= reg_data._size;
+    vec_spill_helper(__ code(), false /* is_load */, _spill_offset, opto_reg, ideal_reg, tty);
+  }
+
+  void xmm_register_restore(const XMMRegisterData& reg_data) {
+    const OptoReg::Name opto_reg = OptoReg::as_OptoReg(reg_data._reg->as_VMReg());
+    const uint ideal_reg = xmm_ideal_reg_for_size(reg_data._size);
+    vec_spill_helper(__ code(), true /* is_load */, _spill_offset, opto_reg, ideal_reg, tty);
+    _spill_offset += reg_data._size;
+  }
+
+  void gp_register_save(Register reg) {
+    _spill_offset -= 8;
+    __ movq(Address(rsp, _spill_offset), reg);
+  }
+
+  void opmask_register_save(KRegister reg) {
+    _spill_offset -= 8;
+    __ kmov(Address(rsp, _spill_offset), reg);
+  }
+
+  void gp_register_restore(Register reg) {
+    __ movq(reg, Address(rsp, _spill_offset));
+    _spill_offset += 8;
+  }
+
+  void opmask_register_restore(KRegister reg) {
+    __ kmov(reg, Address(rsp, _spill_offset));
+    _spill_offset += 8;
+  }
+
+  void initialize(XLoadBarrierStubC2* stub) {
+    // Create mask of caller saved registers that need to
+    // be saved/restored if live
+    RegMask caller_saved;
+    caller_saved.Insert(OptoReg::as_OptoReg(rax->as_VMReg()));
+    caller_saved.Insert(OptoReg::as_OptoReg(rcx->as_VMReg()));
+    caller_saved.Insert(OptoReg::as_OptoReg(rdx->as_VMReg()));
+    caller_saved.Insert(OptoReg::as_OptoReg(rsi->as_VMReg()));
+    caller_saved.Insert(OptoReg::as_OptoReg(rdi->as_VMReg()));
+    caller_saved.Insert(OptoReg::as_OptoReg(r8->as_VMReg()));
+    caller_saved.Insert(OptoReg::as_OptoReg(r9->as_VMReg()));
+    caller_saved.Insert(OptoReg::as_OptoReg(r10->as_VMReg()));
+    caller_saved.Insert(OptoReg::as_OptoReg(r11->as_VMReg()));
+    caller_saved.Remove(OptoReg::as_OptoReg(stub->ref()->as_VMReg()));
+
+    // Create mask of live registers
+    RegMask live = stub->live();
+    if (stub->tmp() != noreg) {
+      live.Insert(OptoReg::as_OptoReg(stub->tmp()->as_VMReg()));
+    }
+
+    int gp_spill_size = 0;
+    int opmask_spill_size = 0;
+    int xmm_spill_size = 0;
+
+    // Record registers that needs to be saved/restored
+    RegMaskIterator rmi(live);
+    while (rmi.has_next()) {
+      const OptoReg::Name opto_reg = rmi.next();
+      const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
+
+      if (vm_reg->is_Register()) {
+        if (caller_saved.Member(opto_reg)) {
+          _gp_registers.append(vm_reg->as_Register());
+          gp_spill_size += 8;
+        }
+      } else if (vm_reg->is_KRegister()) {
+        // All opmask registers are caller saved, thus spill the ones
+        // which are live.
+        if (_opmask_registers.find(vm_reg->as_KRegister()) == -1) {
+          _opmask_registers.append(vm_reg->as_KRegister());
+          opmask_spill_size += 8;
+        }
+      } else if (vm_reg->is_XMMRegister()) {
+        // We encode in the low order 4 bits of the opto_reg, how large part of the register is live
+        const VMReg vm_reg_base = OptoReg::as_VMReg(opto_reg & ~15);
+        const int reg_size = xmm_slot_size(opto_reg);
+        const XMMRegisterData reg_data = { vm_reg_base->as_XMMRegister(), reg_size };
+        const int reg_index = _xmm_registers.find(reg_data);
+        if (reg_index == -1) {
+          // Not previously appended
+          _xmm_registers.append(reg_data);
+          xmm_spill_size += reg_size;
+        } else {
+          // Previously appended, update size
+          const int reg_size_prev = _xmm_registers.at(reg_index)._size;
+          if (reg_size > reg_size_prev) {
+            _xmm_registers.at_put(reg_index, reg_data);
+            xmm_spill_size += reg_size - reg_size_prev;
+          }
+        }
+      } else {
+        fatal("Unexpected register type");
+      }
+    }
+
+    // Sort by size, largest first
+    _xmm_registers.sort(xmm_compare_register_size);
+
+    // On Windows, the caller reserves stack space for spilling register arguments
+    const int arg_spill_size = frame::arg_reg_save_area_bytes;
+
+    // Stack pointer must be 16 bytes aligned for the call
+    _spill_offset = _spill_size = align_up(xmm_spill_size + gp_spill_size + opmask_spill_size + arg_spill_size, 16);
+  }
+
+public:
+  XSaveLiveRegisters(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
+      _masm(masm),
+      _gp_registers(),
+      _opmask_registers(),
+      _xmm_registers(),
+      _spill_size(0),
+      _spill_offset(0) {
+
+    //
+    // Stack layout after registers have been spilled:
+    //
+    // | ...            | original rsp, 16 bytes aligned
+    // ------------------
+    // | zmm0 high      |
+    // | ...            |
+    // | zmm0 low       | 16 bytes aligned
+    // | ...            |
+    // | ymm1 high      |
+    // | ...            |
+    // | ymm1 low       | 16 bytes aligned
+    // | ...            |
+    // | xmmN high      |
+    // | ...            |
+    // | xmmN low       | 8 bytes aligned
+    // | reg0           | 8 bytes aligned
+    // | reg1           |
+    // | ...            |
+    // | regN           | new rsp, if 16 bytes aligned
+    // | <padding>      | else new rsp, 16 bytes aligned
+    // ------------------
+    //
+
+    // Figure out what registers to save/restore
+    initialize(stub);
+
+    // Allocate stack space
+    if (_spill_size > 0) {
+      __ subptr(rsp, _spill_size);
+    }
+
+    // Save XMM/YMM/ZMM registers
+    for (int i = 0; i < _xmm_registers.length(); i++) {
+      xmm_register_save(_xmm_registers.at(i));
+    }
+
+    if (xmm_needs_vzeroupper()) {
+      __ vzeroupper();
+    }
+
+    // Save general purpose registers
+    for (int i = 0; i < _gp_registers.length(); i++) {
+      gp_register_save(_gp_registers.at(i));
+    }
+
+    // Save opmask registers
+    for (int i = 0; i < _opmask_registers.length(); i++) {
+      opmask_register_save(_opmask_registers.at(i));
+    }
+  }
+
+  ~XSaveLiveRegisters() {
+    // Restore opmask registers
+    for (int i = _opmask_registers.length() - 1; i >= 0; i--) {
+      opmask_register_restore(_opmask_registers.at(i));
+    }
+
+    // Restore general purpose registers
+    for (int i = _gp_registers.length() - 1; i >= 0; i--) {
+      gp_register_restore(_gp_registers.at(i));
+    }
+
+    __ vzeroupper();
+
+    // Restore XMM/YMM/ZMM registers
+    for (int i = _xmm_registers.length() - 1; i >= 0; i--) {
+      xmm_register_restore(_xmm_registers.at(i));
+    }
+
+    // Free stack space
+    if (_spill_size > 0) {
+      __ addptr(rsp, _spill_size);
+    }
+  }
+};
+
+class XSetupArguments {
+private:
+  MacroAssembler* const _masm;
+  const Register        _ref;
+  const Address         _ref_addr;
+
+public:
+  XSetupArguments(MacroAssembler* masm, XLoadBarrierStubC2* stub) :
+      _masm(masm),
+      _ref(stub->ref()),
+      _ref_addr(stub->ref_addr()) {
+
+    // Setup arguments
+    if (_ref_addr.base() == noreg) {
+      // No self healing
+      if (_ref != c_rarg0) {
+        __ movq(c_rarg0, _ref);
+      }
+      __ xorq(c_rarg1, c_rarg1);
+    } else {
+      // Self healing
+      if (_ref == c_rarg0) {
+        __ lea(c_rarg1, _ref_addr);
+      } else if (_ref != c_rarg1) {
+        __ lea(c_rarg1, _ref_addr);
+        __ movq(c_rarg0, _ref);
+      } else if (_ref_addr.base() != c_rarg0 && _ref_addr.index() != c_rarg0) {
+        __ movq(c_rarg0, _ref);
+        __ lea(c_rarg1, _ref_addr);
+      } else {
+        __ xchgq(c_rarg0, c_rarg1);
+        if (_ref_addr.base() == c_rarg0) {
+          __ lea(c_rarg1, Address(c_rarg1, _ref_addr.index(), _ref_addr.scale(), _ref_addr.disp()));
+        } else if (_ref_addr.index() == c_rarg0) {
+          __ lea(c_rarg1, Address(_ref_addr.base(), c_rarg1, _ref_addr.scale(), _ref_addr.disp()));
+        } else {
+          ShouldNotReachHere();
+        }
+      }
+    }
+  }
+
+  ~XSetupArguments() {
+    // Transfer result
+    if (_ref != rax) {
+      __ movq(_ref, rax);
+    }
+  }
+};
+
+#undef __
+#define __ masm->
+
+void XBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, XLoadBarrierStubC2* stub) const {
+  BLOCK_COMMENT("XLoadBarrierStubC2");
+
+  // Stub entry
+  __ bind(*stub->entry());
+
+  {
+    XSaveLiveRegisters save_live_registers(masm, stub);
+    XSetupArguments setup_arguments(masm, stub);
+    __ call(RuntimeAddress(stub->slow_path()));
+  }
+
+  // Stub exit
+  __ jmp(*stub->continuation());
+}
+
+#endif // COMPILER2
+
+#undef __
+#define __ masm->
+
+void XBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
+  // Check if metadata bits indicate a bad oop
+  __ testptr(obj, Address(r15_thread, XThreadLocalData::address_bad_mask_offset()));
+  __ jcc(Assembler::notZero, error);
+  BarrierSetAssembler::check_oop(masm, obj, tmp1, tmp2, error);
+}
+
+#undef __
diff --git a/src/hotspot/cpu/x86/gc/x/xBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/x/xBarrierSetAssembler_x86.hpp
new file mode 100644
index 00000000000..52034ab786e
--- /dev/null
+++ b/src/hotspot/cpu/x86/gc/x/xBarrierSetAssembler_x86.hpp
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_X86_GC_X_XBARRIERSETASSEMBLER_X86_HPP
+#define CPU_X86_GC_X_XBARRIERSETASSEMBLER_X86_HPP
+
+#include "code/vmreg.hpp"
+#include "oops/accessDecorators.hpp"
+#ifdef COMPILER2
+#include "opto/optoreg.hpp"
+#endif // COMPILER2
+
+class MacroAssembler;
+
+#ifdef COMPILER1
+class LIR_Assembler;
+class LIR_Opr;
+class StubAssembler;
+#endif // COMPILER1
+
+#ifdef COMPILER2
+class Node;
+#endif // COMPILER2
+
+#ifdef COMPILER1
+class XLoadBarrierStubC1;
+#endif // COMPILER1
+
+#ifdef COMPILER2
+class XLoadBarrierStubC2;
+#endif // COMPILER2
+
+class XBarrierSetAssembler : public XBarrierSetAssemblerBase {
+public:
+  virtual void load_at(MacroAssembler* masm,
+                       DecoratorSet decorators,
+                       BasicType type,
+                       Register dst,
+                       Address src,
+                       Register tmp1,
+                       Register tmp_thread);
+
+#ifdef ASSERT
+  virtual void store_at(MacroAssembler* masm,
+                        DecoratorSet decorators,
+                        BasicType type,
+                        Address dst,
+                        Register src,
+                        Register tmp1,
+                        Register tmp2,
+                        Register tmp3);
+#endif // ASSERT
+
+  virtual void arraycopy_prologue(MacroAssembler* masm,
+                                  DecoratorSet decorators,
+                                  BasicType type,
+                                  Register src,
+                                  Register dst,
+                                  Register count);
+
+  virtual void try_resolve_jobject_in_native(MacroAssembler* masm,
+                                             Register jni_env,
+                                             Register obj,
+                                             Register tmp,
+                                             Label& slowpath);
+
+#ifdef COMPILER1
+  void generate_c1_load_barrier_test(LIR_Assembler* ce,
+                                     LIR_Opr ref) const;
+
+  void generate_c1_load_barrier_stub(LIR_Assembler* ce,
+                                     XLoadBarrierStubC1* stub) const;
+
+  void generate_c1_load_barrier_runtime_stub(StubAssembler* sasm,
+                                             DecoratorSet decorators) const;
+#endif // COMPILER1
+
+#ifdef COMPILER2
+  OptoReg::Name refine_register(const Node* node,
+                                OptoReg::Name opto_reg);
+
+  void generate_c2_load_barrier_stub(MacroAssembler* masm,
+                                     XLoadBarrierStubC2* stub) const;
+#endif // COMPILER2
+
+  void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
+};
+
+#endif // CPU_X86_GC_X_XBARRIERSETASSEMBLER_X86_HPP
diff --git a/src/hotspot/cpu/x86/gc/z/zGlobals_x86.cpp b/src/hotspot/cpu/x86/gc/x/xGlobals_x86.cpp
similarity index 97%
rename from src/hotspot/cpu/x86/gc/z/zGlobals_x86.cpp
rename to src/hotspot/cpu/x86/gc/x/xGlobals_x86.cpp
index 875eb8855dd..baa99ddd60d 100644
--- a/src/hotspot/cpu/x86/gc/z/zGlobals_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/x/xGlobals_x86.cpp
@@ -23,7 +23,7 @@
 
 #include "precompiled.hpp"
 #include "gc/shared/gc_globals.hpp"
-#include "gc/z/zGlobals.hpp"
+#include "gc/x/xGlobals.hpp"
 #include "runtime/globals.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/powerOfTwo.hpp"
@@ -136,14 +136,14 @@
 //  * 63-48 Fixed (16-bits, always zero)
 //
 
-size_t ZPlatformAddressOffsetBits() {
+size_t XPlatformAddressOffsetBits() {
   const size_t min_address_offset_bits = 42; // 4TB
   const size_t max_address_offset_bits = 44; // 16TB
-  const size_t address_offset = round_up_power_of_2(MaxHeapSize * ZVirtualToPhysicalRatio);
+  const size_t address_offset = round_up_power_of_2(MaxHeapSize * XVirtualToPhysicalRatio);
   const size_t address_offset_bits = log2i_exact(address_offset);
   return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
 }
 
-size_t ZPlatformAddressMetadataShift() {
-  return ZPlatformAddressOffsetBits();
+size_t XPlatformAddressMetadataShift() {
+  return XPlatformAddressOffsetBits();
 }
diff --git a/src/hotspot/cpu/x86/gc/x/xGlobals_x86.hpp b/src/hotspot/cpu/x86/gc/x/xGlobals_x86.hpp
new file mode 100644
index 00000000000..dd00d4ddadc
--- /dev/null
+++ b/src/hotspot/cpu/x86/gc/x/xGlobals_x86.hpp
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_X86_GC_X_XGLOBALS_X86_HPP
+#define CPU_X86_GC_X_XGLOBALS_X86_HPP
+
+const size_t XPlatformHeapViews        = 3;
+const size_t XPlatformCacheLineSize    = 64;
+
+size_t XPlatformAddressOffsetBits();
+size_t XPlatformAddressMetadataShift();
+
+#endif // CPU_X86_GC_X_XGLOBALS_X86_HPP
diff --git a/src/hotspot/cpu/x86/gc/x/x_x86_64.ad b/src/hotspot/cpu/x86/gc/x/x_x86_64.ad
new file mode 100644
index 00000000000..c33a994a4b8
--- /dev/null
+++ b/src/hotspot/cpu/x86/gc/x/x_x86_64.ad
@@ -0,0 +1,158 @@
+//
+// Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+//
+// This code is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License version 2 only, as
+// published by the Free Software Foundation.
+//
+// This code is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+// version 2 for more details (a copy is included in the LICENSE file that
+// accompanied this code).
+//
+// You should have received a copy of the GNU General Public License version
+// 2 along with this work; if not, write to the Free Software Foundation,
+// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+//
+// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+// or visit www.oracle.com if you need additional information or have any
+// questions.
+//
+
+source_hpp %{
+
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/c2/xBarrierSetC2.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+
+%}
+
+source %{
+
+#include "c2_intelJccErratum_x86.hpp"
+
+static void x_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data) {
+  if (barrier_data == XLoadBarrierElided) {
+    return;
+  }
+  XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
+  {
+    IntelJccErratumAlignment intel_alignment(_masm, 10 /* jcc_size */);
+    __ testptr(ref, Address(r15_thread, XThreadLocalData::address_bad_mask_offset()));
+    __ jcc(Assembler::notZero, *stub->entry());
+  }
+  __ bind(*stub->continuation());
+}
+
+static void x_load_barrier_cmpxchg(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, Label& good) {
+  XLoadBarrierStubC2* const stub = XLoadBarrierStubC2::create(node, ref_addr, ref, tmp, XLoadBarrierStrong);
+  {
+    IntelJccErratumAlignment intel_alignment(_masm, 10 /* jcc_size */);
+    __ testptr(ref, Address(r15_thread, XThreadLocalData::address_bad_mask_offset()));
+    __ jcc(Assembler::zero, good);
+  }
+  {
+    IntelJccErratumAlignment intel_alignment(_masm, 5 /* jcc_size */);
+    __ jmp(*stub->entry());
+  }
+  __ bind(*stub->continuation());
+}
+
+static void x_cmpxchg_common(MacroAssembler& _masm, const MachNode* node, Register mem_reg, Register newval, Register tmp) {
+  // Compare value (oldval) is in rax
+   const Address mem = Address(mem_reg, 0);
+
+  if (node->barrier_data() != XLoadBarrierElided) {
+    __ movptr(tmp, rax);
+  }
+
+  __ lock();
+  __ cmpxchgptr(newval, mem);
+
+  if (node->barrier_data() != XLoadBarrierElided) {
+    Label good;
+    x_load_barrier_cmpxchg(_masm, node, mem, rax, tmp, good);
+    __ movptr(rax, tmp);
+    __ lock();
+    __ cmpxchgptr(newval, mem);
+    __ bind(good);
+  }
+}
+
+%}
+
+// Load Pointer
+instruct xLoadP(rRegP dst, memory mem, rFlagsReg cr)
+%{
+  predicate(UseZGC && !ZGenerational && n->as_Load()->barrier_data() != 0);
+  match(Set dst (LoadP mem));
+  effect(KILL cr, TEMP dst);
+
+  ins_cost(125);
+
+  format %{ "movq     $dst, $mem" %}
+
+  ins_encode %{
+    __ movptr($dst$$Register, $mem$$Address);
+    x_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, barrier_data());
+  %}
+
+  ins_pipe(ialu_reg_mem);
+%}
+
+instruct xCompareAndExchangeP(indirect mem, rax_RegP oldval, rRegP newval, rRegP tmp, rFlagsReg cr) %{
+  match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
+  predicate(UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
+  effect(KILL cr, TEMP tmp);
+
+  format %{ "lock\n\t"
+            "cmpxchgq $newval, $mem" %}
+
+  ins_encode %{
+    precond($oldval$$Register == rax);
+    x_cmpxchg_common(_masm, this, $mem$$Register, $newval$$Register, $tmp$$Register);
+  %}
+
+  ins_pipe(pipe_cmpxchg);
+%}
+
+instruct xCompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp, rFlagsReg cr, rax_RegP oldval) %{
+  match(Set res (CompareAndSwapP mem (Binary oldval newval)));
+  match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
+  predicate(UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() == XLoadBarrierStrong);
+  effect(KILL cr, KILL oldval, TEMP tmp);
+
+  format %{ "lock\n\t"
+            "cmpxchgq $newval, $mem\n\t"
+            "sete     $res\n\t"
+            "movzbl   $res, $res" %}
+
+  ins_encode %{
+    precond($oldval$$Register == rax);
+    x_cmpxchg_common(_masm, this, $mem$$Register, $newval$$Register, $tmp$$Register);
+    if (barrier_data() != XLoadBarrierElided) {
+      __ cmpptr($tmp$$Register, rax);
+    }
+    __ setb(Assembler::equal, $res$$Register);
+    __ movzbl($res$$Register, $res$$Register);
+  %}
+
+  ins_pipe(pipe_cmpxchg);
+%}
+
+instruct xXChgP(indirect mem, rRegP newval, rFlagsReg cr) %{
+  match(Set newval (GetAndSetP mem newval));
+  predicate(UseZGC && !ZGenerational && n->as_LoadStore()->barrier_data() != 0);
+  effect(KILL cr);
+
+  format %{ "xchgq    $newval, $mem" %}
+
+  ins_encode %{
+    __ xchgptr($newval$$Register, Address($mem$$Register, 0));
+    x_load_barrier(_masm, this, Address(noreg, 0), $newval$$Register, noreg /* tmp */, barrier_data());
+  %}
+
+  ins_pipe(pipe_cmpxchg);
+%}
diff --git a/src/hotspot/cpu/x86/gc/z/zAddress_x86.cpp b/src/hotspot/cpu/x86/gc/z/zAddress_x86.cpp
new file mode 100644
index 00000000000..ed177f37e0d
--- /dev/null
+++ b/src/hotspot/cpu/x86/gc/z/zAddress_x86.cpp
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+size_t ZPointerLoadShift;
+
+size_t ZPlatformAddressOffsetBits() {
+  const size_t min_address_offset_bits = 42; // 4TB
+  const size_t max_address_offset_bits = 44; // 16TB
+  const size_t address_offset = round_up_power_of_2(MaxHeapSize * ZVirtualToPhysicalRatio);
+  const size_t address_offset_bits = log2i_exact(address_offset);
+  return clamp(address_offset_bits, min_address_offset_bits, max_address_offset_bits);
+}
+
+size_t ZPlatformAddressHeapBaseShift() {
+  return ZPlatformAddressOffsetBits();
+}
+
+void ZGlobalsPointers::pd_set_good_masks() {
+  ZPointerLoadShift = ZPointer::load_shift_lookup(ZPointerLoadGoodMask);
+}
diff --git a/src/hotspot/cpu/x86/gc/z/zAddress_x86.hpp b/src/hotspot/cpu/x86/gc/z/zAddress_x86.hpp
new file mode 100644
index 00000000000..e71333f6068
--- /dev/null
+++ b/src/hotspot/cpu/x86/gc/z/zAddress_x86.hpp
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_X86_GC_Z_ZADDRESS_X86_HPP
+#define CPU_X86_GC_Z_ZADDRESS_X86_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+extern size_t ZPointerLoadShift;
+
+size_t ZPlatformAddressOffsetBits();
+size_t ZPlatformAddressHeapBaseShift();
+
+#endif // CPU_X86_GC_Z_ZADDRESS_X86_HPP
diff --git a/src/hotspot/cpu/x86/gc/z/zAddress_x86.inline.hpp b/src/hotspot/cpu/x86/gc/z/zAddress_x86.inline.hpp
new file mode 100644
index 00000000000..e0be0639594
--- /dev/null
+++ b/src/hotspot/cpu/x86/gc/z/zAddress_x86.inline.hpp
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef CPU_X86_GC_Z_ZADDRESS_X86_INLINE_HPP
+#define CPU_X86_GC_Z_ZADDRESS_X86_INLINE_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+inline uintptr_t ZPointer::remap_bits(uintptr_t colored) {
+  return colored & ZPointerRemappedMask;
+}
+
+inline constexpr int ZPointer::load_shift_lookup(uintptr_t value) {
+  const size_t index = load_shift_lookup_index(value);
+  assert(index == 0 || is_power_of_2(index), "Incorrect load shift: " SIZE_FORMAT, index);
+  return ZPointerLoadShiftTable[index];
+}
+
+#endif // CPU_X86_GC_Z_ZADDRESS_X86_INLINE_HPP
diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
index 0f4e62341a5..392e661ed8a 100644
--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.cpp
@@ -25,12 +25,14 @@
 #include "asm/macroAssembler.inline.hpp"
 #include "code/codeBlob.hpp"
 #include "code/vmreg.inline.hpp"
+#include "gc/z/zAddress.hpp"
 #include "gc/z/zBarrier.inline.hpp"
 #include "gc/z/zBarrierSet.hpp"
 #include "gc/z/zBarrierSetAssembler.hpp"
 #include "gc/z/zBarrierSetRuntime.hpp"
 #include "gc/z/zThreadLocalData.hpp"
 #include "memory/resourceArea.hpp"
+#include "runtime/jniHandles.hpp"
 #include "runtime/sharedRuntime.hpp"
 #include "utilities/macros.hpp"
 #ifdef COMPILER1
@@ -39,7 +41,9 @@
 #include "gc/z/c1/zBarrierSetC1.hpp"
 #endif // COMPILER1
 #ifdef COMPILER2
+#include "c2_intelJccErratum_x86.hpp"
 #include "gc/z/c2/zBarrierSetC2.hpp"
+#include "opto/output.hpp"
 #endif // COMPILER2
 
 #ifdef PRODUCT
@@ -51,6 +55,142 @@
 #undef __
 #define __ masm->
 
+ZBarrierSetAssembler::ZBarrierSetAssembler()
+  : _load_bad_relocations(),
+    _store_bad_relocations(),
+    _store_good_relocations() {
+}
+
+enum class ZXMMSpillMode {
+  none,
+  avx128,
+  avx256
+};
+
+// Helper for saving and restoring registers across a runtime call that does
+// not have any live vector registers.
+class ZRuntimeCallSpill {
+private:
+  const ZXMMSpillMode _xmm_spill_mode;
+  const int _xmm_size;
+  const int _xmm_spill_size;
+  MacroAssembler* _masm;
+  Register _result;
+
+  void save() {
+    MacroAssembler* masm = _masm;
+    __ push(rax);
+    __ push(rcx);
+    __ push(rdx);
+    __ push(rdi);
+    __ push(rsi);
+    __ push(r8);
+    __ push(r9);
+    __ push(r10);
+    __ push(r11);
+
+    if (_xmm_spill_size != 0) {
+      __ subptr(rsp, _xmm_spill_size);
+      if (_xmm_spill_mode == ZXMMSpillMode::avx128) {
+        __ movdqu(Address(rsp, _xmm_size * 7), xmm7);
+        __ movdqu(Address(rsp, _xmm_size * 6), xmm6);
+        __ movdqu(Address(rsp, _xmm_size * 5), xmm5);
+        __ movdqu(Address(rsp, _xmm_size * 4), xmm4);
+        __ movdqu(Address(rsp, _xmm_size * 3), xmm3);
+        __ movdqu(Address(rsp, _xmm_size * 2), xmm2);
+        __ movdqu(Address(rsp, _xmm_size * 1), xmm1);
+        __ movdqu(Address(rsp, _xmm_size * 0), xmm0);
+      } else {
+        assert(_xmm_spill_mode == ZXMMSpillMode::avx256, "AVX support ends at avx256");
+        __ vmovdqu(Address(rsp, _xmm_size * 7), xmm7);
+        __ vmovdqu(Address(rsp, _xmm_size * 6), xmm6);
+        __ vmovdqu(Address(rsp, _xmm_size * 5), xmm5);
+        __ vmovdqu(Address(rsp, _xmm_size * 4), xmm4);
+        __ vmovdqu(Address(rsp, _xmm_size * 3), xmm3);
+        __ vmovdqu(Address(rsp, _xmm_size * 2), xmm2);
+        __ vmovdqu(Address(rsp, _xmm_size * 1), xmm1);
+        __ vmovdqu(Address(rsp, _xmm_size * 0), xmm0);
+      }
+    }
+  }
+
+  void restore() {
+    MacroAssembler* masm = _masm;
+    if (_xmm_spill_size != 0) {
+      if (_xmm_spill_mode == ZXMMSpillMode::avx128) {
+        __ movdqu(xmm0, Address(rsp, _xmm_size * 0));
+        __ movdqu(xmm1, Address(rsp, _xmm_size * 1));
+        __ movdqu(xmm2, Address(rsp, _xmm_size * 2));
+        __ movdqu(xmm3, Address(rsp, _xmm_size * 3));
+        __ movdqu(xmm4, Address(rsp, _xmm_size * 4));
+        __ movdqu(xmm5, Address(rsp, _xmm_size * 5));
+        __ movdqu(xmm6, Address(rsp, _xmm_size * 6));
+        __ movdqu(xmm7, Address(rsp, _xmm_size * 7));
+      } else {
+        assert(_xmm_spill_mode == ZXMMSpillMode::avx256, "AVX support ends at avx256");
+        __ vmovdqu(xmm0, Address(rsp, _xmm_size * 0));
+        __ vmovdqu(xmm1, Address(rsp, _xmm_size * 1));
+        __ vmovdqu(xmm2, Address(rsp, _xmm_size * 2));
+        __ vmovdqu(xmm3, Address(rsp, _xmm_size * 3));
+        __ vmovdqu(xmm4, Address(rsp, _xmm_size * 4));
+        __ vmovdqu(xmm5, Address(rsp, _xmm_size * 5));
+        __ vmovdqu(xmm6, Address(rsp, _xmm_size * 6));
+        __ vmovdqu(xmm7, Address(rsp, _xmm_size * 7));
+      }
+      __ addptr(rsp, _xmm_spill_size);
+    }
+
+    __ pop(r11);
+    __ pop(r10);
+    __ pop(r9);
+    __ pop(r8);
+    __ pop(rsi);
+    __ pop(rdi);
+    __ pop(rdx);
+    __ pop(rcx);
+    if (_result == noreg) {
+      __ pop(rax);
+    } else if (_result == rax) {
+      __ addptr(rsp, wordSize);
+    } else {
+      __ movptr(_result, rax);
+      __ pop(rax);
+    }
+  }
+
+  static int compute_xmm_size(ZXMMSpillMode spill_mode) {
+    switch (spill_mode) {
+      case ZXMMSpillMode::none:
+        return 0;
+      case ZXMMSpillMode::avx128:
+        return wordSize * 2;
+      case ZXMMSpillMode::avx256:
+        return wordSize * 4;
+      default:
+        ShouldNotReachHere();
+        return 0;
+    }
+  }
+
+public:
+  ZRuntimeCallSpill(MacroAssembler* masm, Register result, ZXMMSpillMode spill_mode)
+    : _xmm_spill_mode(spill_mode),
+      _xmm_size(compute_xmm_size(spill_mode)),
+      _xmm_spill_size(_xmm_size * Argument::n_float_register_parameters_j),
+      _masm(masm),
+      _result(result) {
+    // We may end up here from generate_native_wrapper, then the method may have
+    // floats as arguments, and we must spill them before calling the VM runtime
+    // leaf. From the interpreter all floats are passed on the stack.
+    assert(Argument::n_float_register_parameters_j == 8, "Assumption");
+    save();
+  }
+
+  ~ZRuntimeCallSpill() {
+    restore();
+  }
+};
+
 static void call_vm(MacroAssembler* masm,
                     address entry_point,
                     Register arg0,
@@ -101,6 +241,7 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
   assert_different_registers(dst, scratch);
 
   Label done;
+  Label uncolor;
 
   //
   // Fast Path
@@ -112,70 +253,44 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
   // Load oop at address
   __ movptr(dst, Address(scratch, 0));
 
+  const bool on_non_strong =
+      (decorators & ON_WEAK_OOP_REF) != 0 ||
+      (decorators & ON_PHANTOM_OOP_REF) != 0;
+
   // Test address bad mask
-  __ testptr(dst, address_bad_mask_from_thread(r15_thread));
-  __ jcc(Assembler::zero, done);
+  if (on_non_strong) {
+    __ testptr(dst, mark_bad_mask_from_thread(r15_thread));
+  } else {
+    __ testptr(dst, load_bad_mask_from_thread(r15_thread));
+  }
+
+  __ jcc(Assembler::zero, uncolor);
 
   //
   // Slow path
   //
 
-  // Save registers
-  __ push(rax);
-  __ push(rcx);
-  __ push(rdx);
-  __ push(rdi);
-  __ push(rsi);
-  __ push(r8);
-  __ push(r9);
-  __ push(r10);
-  __ push(r11);
-
-  // We may end up here from generate_native_wrapper, then the method may have
-  // floats as arguments, and we must spill them before calling the VM runtime
-  // leaf. From the interpreter all floats are passed on the stack.
-  assert(Argument::n_float_register_parameters_j == 8, "Assumption");
-  const int xmm_size = wordSize * 2;
-  const int xmm_spill_size = xmm_size * Argument::n_float_register_parameters_j;
-  __ subptr(rsp, xmm_spill_size);
-  __ movdqu(Address(rsp, xmm_size * 7), xmm7);
-  __ movdqu(Address(rsp, xmm_size * 6), xmm6);
-  __ movdqu(Address(rsp, xmm_size * 5), xmm5);
-  __ movdqu(Address(rsp, xmm_size * 4), xmm4);
-  __ movdqu(Address(rsp, xmm_size * 3), xmm3);
-  __ movdqu(Address(rsp, xmm_size * 2), xmm2);
-  __ movdqu(Address(rsp, xmm_size * 1), xmm1);
-  __ movdqu(Address(rsp, xmm_size * 0), xmm0);
-
-  // Call VM
-  call_vm(masm, ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), dst, scratch);
-
-  __ movdqu(xmm0, Address(rsp, xmm_size * 0));
-  __ movdqu(xmm1, Address(rsp, xmm_size * 1));
-  __ movdqu(xmm2, Address(rsp, xmm_size * 2));
-  __ movdqu(xmm3, Address(rsp, xmm_size * 3));
-  __ movdqu(xmm4, Address(rsp, xmm_size * 4));
-  __ movdqu(xmm5, Address(rsp, xmm_size * 5));
-  __ movdqu(xmm6, Address(rsp, xmm_size * 6));
-  __ movdqu(xmm7, Address(rsp, xmm_size * 7));
-  __ addptr(rsp, xmm_spill_size);
-
-  __ pop(r11);
-  __ pop(r10);
-  __ pop(r9);
-  __ pop(r8);
-  __ pop(rsi);
-  __ pop(rdi);
-  __ pop(rdx);
-  __ pop(rcx);
-
-  if (dst == rax) {
-    __ addptr(rsp, wordSize);
-  } else {
-    __ movptr(dst, rax);
-    __ pop(rax);
+  {
+    // Call VM
+    ZRuntimeCallSpill rcs(masm, dst, ZXMMSpillMode::avx128);
+    call_vm(masm, ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators), dst, scratch);
   }
 
+  // Slow-path has already uncolored
+  __ jmp(done);
+
+  __ bind(uncolor);
+
+  __ movptr(scratch, rcx); // Save rcx because shrq needs shift in rcx
+  __ movptr(rcx, ExternalAddress((address)&ZPointerLoadShift));
+  if (dst == rcx) {
+    // Dst was rcx which is saved in scratch because shrq needs rcx for shift
+    __ shrq(scratch);
+  } else {
+    __ shrq(dst);
+  }
+  __ movptr(rcx, scratch); // restore rcx
+
   __ bind(done);
 
   // Restore scratch register
@@ -186,7 +301,208 @@ void ZBarrierSetAssembler::load_at(MacroAssembler* masm,
   BLOCK_COMMENT("} ZBarrierSetAssembler::load_at");
 }
 
-#ifdef ASSERT
+static void emit_store_fast_path_check(MacroAssembler* masm, Address ref_addr, bool is_atomic, Label& medium_path) {
+  if (is_atomic) {
+    // Atomic operations must ensure that the contents of memory are store-good before
+    // an atomic operation can execute.
+    // A not relocatable object could have spurious raw null pointers in its fields after
+    // getting promoted to the old generation.
+    __ cmpw(ref_addr, barrier_Relocation::unpatched);
+    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodAfterCmp);
+  } else {
+    // Stores on relocatable objects never need to deal with raw null pointers in fields.
+    // Raw null pointers may only exist in the young generation, as they get pruned when
+    // the object is relocated to old. And no pre-write barrier needs to perform any action
+    // in the young generation.
+    __ Assembler::testl(ref_addr, barrier_Relocation::unpatched);
+    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreBadAfterTest);
+  }
+  __ jcc(Assembler::notEqual, medium_path);
+}
+
+static int store_fast_path_check_size(MacroAssembler* masm, Address ref_addr, bool is_atomic, Label& medium_path) {
+  if (!VM_Version::has_intel_jcc_erratum()) {
+    return 0;
+  }
+  int size = 0;
+#ifdef COMPILER2
+  bool in_scratch_emit_size = masm->code_section()->scratch_emit();
+  if (!in_scratch_emit_size) {
+    // Temporarily register as scratch buffer so that relocations don't register
+    masm->code_section()->set_scratch_emit();
+  }
+  // First emit the code, to measure its size
+  address insts_end = masm->code_section()->end();
+  // The dummy medium path label is bound after the code emission. This ensures
+  // full size of the generated jcc, which is what the real barrier will have
+  // as well, as it also binds after the emission of the barrier.
+  Label dummy_medium_path;
+  emit_store_fast_path_check(masm, ref_addr, is_atomic, dummy_medium_path);
+  address emitted_end = masm->code_section()->end();
+  size = (int)(intptr_t)(emitted_end - insts_end);
+  __ bind(dummy_medium_path);
+  if (!in_scratch_emit_size) {
+    // Potentially restore scratchyness
+    masm->code_section()->clear_scratch_emit();
+  }
+  // Roll back code, now that we know the size
+  masm->code_section()->set_end(insts_end);
+#endif
+  return size;
+}
+
+static void emit_store_fast_path_check_c2(MacroAssembler* masm, Address ref_addr, bool is_atomic, Label& medium_path) {
+#ifdef COMPILER2
+  // This is a JCC erratum mitigation wrapper for calling the inner check
+  int size = store_fast_path_check_size(masm, ref_addr, is_atomic, medium_path);
+  // Emit JCC erratum mitigation nops with the right size
+  IntelJccErratumAlignment(*masm, size);
+  // Emit the JCC erratum mitigation guarded code
+  emit_store_fast_path_check(masm, ref_addr, is_atomic, medium_path);
+#endif
+}
+
+static bool is_c2_compilation() {
+  CompileTask* task = ciEnv::current()->task();
+  return task != nullptr && is_c2_compile(task->comp_level());
+}
+
+void ZBarrierSetAssembler::store_barrier_fast(MacroAssembler* masm,
+                                              Address ref_addr,
+                                              Register rnew_zaddress,
+                                              Register rnew_zpointer,
+                                              bool in_nmethod,
+                                              bool is_atomic,
+                                              Label& medium_path,
+                                              Label& medium_path_continuation) const {
+  assert_different_registers(ref_addr.base(), rnew_zpointer);
+  assert_different_registers(ref_addr.index(), rnew_zpointer);
+  assert_different_registers(rnew_zaddress, rnew_zpointer);
+
+  if (in_nmethod) {
+    if (is_c2_compilation()) {
+      emit_store_fast_path_check_c2(masm, ref_addr, is_atomic, medium_path);
+    } else {
+      emit_store_fast_path_check(masm, ref_addr, is_atomic, medium_path);
+    }
+    __ bind(medium_path_continuation);
+    if (rnew_zaddress != noreg) {
+      // noreg means null; no need to color
+      __ movptr(rnew_zpointer, rnew_zaddress);
+      __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
+      __ shlq(rnew_zpointer, barrier_Relocation::unpatched);
+      __ orq_imm32(rnew_zpointer, barrier_Relocation::unpatched);
+      __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodAfterOr);
+    }
+  } else {
+    __ movzwq(rnew_zpointer, ref_addr);
+    __ testq(rnew_zpointer, Address(r15_thread, ZThreadLocalData::store_bad_mask_offset()));
+    __ jcc(Assembler::notEqual, medium_path);
+    __ bind(medium_path_continuation);
+    if (rnew_zaddress == noreg) {
+      __ xorptr(rnew_zpointer, rnew_zpointer);
+    } else {
+      __ movptr(rnew_zpointer, rnew_zaddress);
+    }
+    assert_different_registers(rcx, rnew_zpointer);
+    __ push(rcx);
+    __ movptr(rcx, ExternalAddress((address)&ZPointerLoadShift));
+    __ shlq(rnew_zpointer);
+    __ pop(rcx);
+    __ orq(rnew_zpointer, Address(r15_thread, ZThreadLocalData::store_good_mask_offset()));
+  }
+}
+
+static void store_barrier_buffer_add(MacroAssembler* masm,
+                                     Address ref_addr,
+                                     Register tmp1,
+                                     Label& slow_path) {
+  Address buffer(r15_thread, ZThreadLocalData::store_barrier_buffer_offset());
+
+  __ movptr(tmp1, buffer);
+
+  // Combined pointer bump and check if the buffer is disabled or full
+  __ cmpptr(Address(tmp1, ZStoreBarrierBuffer::current_offset()), 0);
+  __ jcc(Assembler::equal, slow_path);
+
+  Register tmp2 = r15_thread;
+  __ push(tmp2);
+
+  // Bump the pointer
+  __ movq(tmp2, Address(tmp1, ZStoreBarrierBuffer::current_offset()));
+  __ subq(tmp2, sizeof(ZStoreBarrierEntry));
+  __ movq(Address(tmp1, ZStoreBarrierBuffer::current_offset()), tmp2);
+
+  // Compute the buffer entry address
+  __ lea(tmp2, Address(tmp1, tmp2, Address::times_1, ZStoreBarrierBuffer::buffer_offset()));
+
+  // Compute and log the store address
+  __ lea(tmp1, ref_addr);
+  __ movptr(Address(tmp2, in_bytes(ZStoreBarrierEntry::p_offset())), tmp1);
+
+  // Load and log the prev value
+  __ movptr(tmp1, Address(tmp1, 0));
+  __ movptr(Address(tmp2, in_bytes(ZStoreBarrierEntry::prev_offset())), tmp1);
+
+  __ pop(tmp2);
+}
+
+void ZBarrierSetAssembler::store_barrier_medium(MacroAssembler* masm,
+                                                Address ref_addr,
+                                                Register tmp,
+                                                bool is_native,
+                                                bool is_atomic,
+                                                Label& medium_path_continuation,
+                                                Label& slow_path,
+                                                Label& slow_path_continuation) const {
+  assert_different_registers(ref_addr.base(), tmp);
+
+  // The reason to end up in the medium path is that the pre-value was not 'good'.
+
+  if (is_native) {
+    __ jmp(slow_path);
+    __ bind(slow_path_continuation);
+    __ jmp(medium_path_continuation);
+  } else if (is_atomic) {
+    // Atomic accesses can get to the medium fast path because the value was a
+    // raw null value. If it was not null, then there is no doubt we need to take a slow path.
+    __ cmpptr(ref_addr, 0);
+    __ jcc(Assembler::notEqual, slow_path);
+
+    // If we get this far, we know there is a young raw null value in the field.
+    // Try to self-heal null values for atomic accesses
+    __ push(rax);
+    __ push(rbx);
+    __ push(rcx);
+
+    __ lea(rcx, ref_addr);
+    __ xorq(rax, rax);
+    __ movptr(rbx, Address(r15, ZThreadLocalData::store_good_mask_offset()));
+
+    __ lock();
+    __ cmpxchgq(rbx, Address(rcx, 0));
+
+    __ pop(rcx);
+    __ pop(rbx);
+    __ pop(rax);
+
+    __ jcc(Assembler::notEqual, slow_path);
+
+    __ bind(slow_path_continuation);
+    __ jmp(medium_path_continuation);
+  } else {
+    // A non-atomic relocatable object won't get to the medium fast path due to a
+    // raw null in the young generation. We only get here because the field is bad.
+    // In this path we don't need any self healing, so we can avoid a runtime call
+    // most of the time by buffering the store barrier to be applied lazily.
+    store_barrier_buffer_add(masm,
+                             ref_addr,
+                             tmp,
+                             slow_path);
+    __ bind(slow_path_continuation);
+    __ jmp(medium_path_continuation);
+  }
+}
 
 void ZBarrierSetAssembler::store_at(MacroAssembler* masm,
                                     DecoratorSet decorators,
@@ -198,27 +514,378 @@ void ZBarrierSetAssembler::store_at(MacroAssembler* masm,
                                     Register tmp3) {
   BLOCK_COMMENT("ZBarrierSetAssembler::store_at {");
 
-  // Verify oop store
+  bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
+
   if (is_reference_type(type)) {
-    // Note that src could be noreg, which means we
-    // are storing null and can skip verification.
-    if (src != noreg) {
+    assert_different_registers(src, tmp1, dst.base(), dst.index());
+
+    if (dest_uninitialized) {
+      assert_different_registers(rcx, tmp1);
+      if (src == noreg) {
+        __ xorq(tmp1, tmp1);
+      } else {
+        __ movptr(tmp1, src);
+      }
+      __ push(rcx);
+      __ movptr(rcx, ExternalAddress((address)&ZPointerLoadShift));
+      __ shlq(tmp1);
+      __ pop(rcx);
+      __ orq(tmp1, Address(r15_thread, ZThreadLocalData::store_good_mask_offset()));
+    } else {
       Label done;
-      __ testptr(src, address_bad_mask_from_thread(r15_thread));
-      __ jcc(Assembler::zero, done);
-      __ stop("Verify oop store failed");
-      __ should_not_reach_here();
+      Label medium;
+      Label medium_continuation;
+      Label slow;
+      Label slow_continuation;
+      store_barrier_fast(masm, dst, src, tmp1, false, false, medium, medium_continuation);
+      __ jmp(done);
+      __ bind(medium);
+      store_barrier_medium(masm,
+                           dst,
+                           tmp1,
+                           false /* is_native */,
+                           false /* is_atomic */,
+                           medium_continuation,
+                           slow,
+                           slow_continuation);
+
+      __ bind(slow);
+      {
+        // Call VM
+        ZRuntimeCallSpill rcs(masm, noreg, ZXMMSpillMode::avx128);
+        __ leaq(c_rarg0, dst);
+        __ MacroAssembler::call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), c_rarg0);
+      }
+
+      __ jmp(slow_continuation);
       __ bind(done);
     }
-  }
 
-  // Store value
-  BarrierSetAssembler::store_at(masm, decorators, type, dst, src, tmp1, tmp2, tmp3);
+    // Store value
+    BarrierSetAssembler::store_at(masm, decorators, type, dst, tmp1, noreg, noreg, noreg);
+  } else {
+    BarrierSetAssembler::store_at(masm, decorators, type, dst, src, noreg, noreg, noreg);
+  }
 
   BLOCK_COMMENT("} ZBarrierSetAssembler::store_at");
 }
 
-#endif // ASSERT
+bool ZBarrierSetAssembler::supports_avx3_masked_arraycopy() {
+  return false;
+}
+
+static void load_arraycopy_masks(MacroAssembler* masm) {
+  // xmm2: load_bad_mask
+  // xmm3: store_bad_mask
+  // xmm4: store_good_mask
+  if (UseAVX >= 2) {
+    __ lea(r10, ExternalAddress((address)&ZPointerVectorLoadBadMask));
+    __ vmovdqu(xmm2, Address(r10, 0));
+    __ lea(r10, ExternalAddress((address)&ZPointerVectorStoreBadMask));
+    __ vmovdqu(xmm3, Address(r10, 0));
+    __ lea(r10, ExternalAddress((address)&ZPointerVectorStoreGoodMask));
+    __ vmovdqu(xmm4, Address(r10, 0));
+  } else {
+    __ lea(r10, ExternalAddress((address)&ZPointerVectorLoadBadMask));
+    __ movdqu(xmm2, Address(r10, 0));
+    __ lea(r10, ExternalAddress((address)&ZPointerVectorStoreBadMask));
+    __ movdqu(xmm3, Address(r10, 0));
+    __ lea(r10, ExternalAddress((address)&ZPointerVectorStoreGoodMask));
+    __ movdqu(xmm4, Address(r10, 0));
+  }
+}
+
+static ZXMMSpillMode compute_arraycopy_spill_mode() {
+  if (UseAVX >= 2) {
+    return ZXMMSpillMode::avx256;
+  } else {
+    return ZXMMSpillMode::avx128;
+  }
+}
+
+void ZBarrierSetAssembler::copy_load_at(MacroAssembler* masm,
+                                        DecoratorSet decorators,
+                                        BasicType type,
+                                        size_t bytes,
+                                        Register dst,
+                                        Address src,
+                                        Register tmp) {
+  if (!is_reference_type(type)) {
+    BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst, src, tmp);
+    return;
+  }
+
+  Label load_done;
+
+  // Load oop at address
+  __ movptr(dst, src);
+
+  // Test address bad mask
+  __ Assembler::testl(dst, (int32_t)(uint32_t)ZPointerLoadBadMask);
+  _load_bad_relocations.append(__ code_section()->end());
+  __ jcc(Assembler::zero, load_done);
+
+  {
+    // Call VM
+    ZRuntimeCallSpill rcs(masm, dst, compute_arraycopy_spill_mode());
+    __ leaq(c_rarg1, src);
+    call_vm(masm, ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_store_good_addr(), dst, c_rarg1);
+  }
+
+  __ bind(load_done);
+
+  // Remove metadata bits so that the store side (vectorized or non-vectorized) can
+  // inject the store-good color with an or instruction.
+  __ andq(dst, _zpointer_address_mask);
+
+  if ((decorators & ARRAYCOPY_CHECKCAST) != 0) {
+    // The checkcast arraycopy needs to be able to dereference the oops in order to perform a typechecks.
+    assert(tmp != rcx, "Surprising choice of temp register");
+    __ movptr(tmp, rcx);
+    __ movptr(rcx, ExternalAddress((address)&ZPointerLoadShift));
+    __ shrq(dst);
+    __ movptr(rcx, tmp);
+  }
+}
+
+void ZBarrierSetAssembler::copy_store_at(MacroAssembler* masm,
+                                         DecoratorSet decorators,
+                                         BasicType type,
+                                         size_t bytes,
+                                         Address dst,
+                                         Register src,
+                                         Register tmp) {
+  if (!is_reference_type(type)) {
+    BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src, tmp);
+    return;
+  }
+
+  bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
+
+  if (!dest_uninitialized) {
+    Label store;
+    Label store_bad;
+    __ Assembler::testl(dst, (int32_t)(uint32_t)ZPointerStoreBadMask);
+    _store_bad_relocations.append(__ code_section()->end());
+    __ jcc(Assembler::zero, store);
+
+    store_barrier_buffer_add(masm, dst, tmp, store_bad);
+    __ jmp(store);
+
+    __ bind(store_bad);
+    {
+      // Call VM
+      ZRuntimeCallSpill rcs(masm, noreg, compute_arraycopy_spill_mode());
+      __ leaq(c_rarg0, dst);
+      __ MacroAssembler::call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), c_rarg0);
+    }
+
+    __ bind(store);
+  }
+
+  if ((decorators & ARRAYCOPY_CHECKCAST) != 0) {
+    assert(tmp != rcx, "Surprising choice of temp register");
+    __ movptr(tmp, rcx);
+    __ movptr(rcx, ExternalAddress((address)&ZPointerLoadShift));
+    __ shlq(src);
+    __ movptr(rcx, tmp);
+  }
+
+  // Color
+  __ orq_imm32(src, (int32_t)(uint32_t)ZPointerStoreGoodMask);
+  _store_good_relocations.append(__ code_section()->end());
+
+  // Store value
+  __ movptr(dst, src);
+}
+
+void ZBarrierSetAssembler::copy_load_at(MacroAssembler* masm,
+                                        DecoratorSet decorators,
+                                        BasicType type,
+                                        size_t bytes,
+                                        XMMRegister dst,
+                                        Address src,
+                                        Register tmp,
+                                        XMMRegister xmm_tmp) {
+  if (!is_reference_type(type)) {
+    BarrierSetAssembler::copy_load_at(masm, decorators, type, bytes, dst, src, tmp, xmm_tmp);
+    return;
+  }
+  Address src0(src.base(), src.index(), src.scale(), src.disp() + 0);
+  Address src1(src.base(), src.index(), src.scale(), src.disp() + 8);
+  Address src2(src.base(), src.index(), src.scale(), src.disp() + 16);
+  Address src3(src.base(), src.index(), src.scale(), src.disp() + 24);
+
+  // Registers set up in the prologue:
+  // xmm2: load_bad_mask
+  // xmm3: store_bad_mask
+  // xmm4: store_good_mask
+
+  if (bytes == 16) {
+    Label done;
+    Label fallback;
+
+    if (UseAVX >= 1) {
+      // Load source vector
+      __ movdqu(dst, src);
+      // Check source load-good
+      __ movdqu(xmm_tmp, dst);
+      __ ptest(xmm_tmp, xmm2);
+      __ jcc(Assembler::notZero, fallback);
+
+      // Remove bad metadata bits
+      __ vpandn(dst, xmm3, dst, Assembler::AVX_128bit);
+      __ jmp(done);
+    }
+
+    __ bind(fallback);
+
+    __ subptr(rsp, wordSize * 2);
+
+    ZBarrierSetAssembler::copy_load_at(masm, decorators, type, 8, tmp, src0, noreg);
+    __ movq(Address(rsp, 0), tmp);
+    ZBarrierSetAssembler::copy_load_at(masm, decorators, type, 8, tmp, src1, noreg);
+    __ movq(Address(rsp, 8), tmp);
+
+    __ movdqu(dst, Address(rsp, 0));
+    __ addptr(rsp, wordSize * 2);
+
+    __ bind(done);
+  } else if (bytes == 32) {
+    Label done;
+    Label fallback;
+    assert(UseAVX >= 2, "Assume that UseAVX >= 2");
+
+    // Load source vector
+    __ vmovdqu(dst, src);
+    // Check source load-good
+    __ vmovdqu(xmm_tmp, dst);
+    __ vptest(xmm_tmp, xmm2, Assembler::AVX_256bit);
+    __ jcc(Assembler::notZero, fallback);
+
+    // Remove bad metadata bits so that the store can colour the pointers with an or instruction.
+    // This makes the fast path and slow path formats look the same, in the sense that they don't
+    // have any of the store bad bits.
+    __ vpandn(dst, xmm3, dst, Assembler::AVX_256bit);
+    __ jmp(done);
+
+    __ bind(fallback);
+
+    __ subptr(rsp, wordSize * 4);
+
+    ZBarrierSetAssembler::copy_load_at(masm, decorators, type, 8, tmp, src0, noreg);
+    __ movq(Address(rsp, 0), tmp);
+    ZBarrierSetAssembler::copy_load_at(masm, decorators, type, 8, tmp, src1, noreg);
+    __ movq(Address(rsp, 8), tmp);
+    ZBarrierSetAssembler::copy_load_at(masm, decorators, type, 8, tmp, src2, noreg);
+    __ movq(Address(rsp, 16), tmp);
+    ZBarrierSetAssembler::copy_load_at(masm, decorators, type, 8, tmp, src3, noreg);
+    __ movq(Address(rsp, 24), tmp);
+
+    __ vmovdqu(dst, Address(rsp, 0));
+    __ addptr(rsp, wordSize * 4);
+
+    __ bind(done);
+  }
+}
+
+void ZBarrierSetAssembler::copy_store_at(MacroAssembler* masm,
+                                         DecoratorSet decorators,
+                                         BasicType type,
+                                         size_t bytes,
+                                         Address dst,
+                                         XMMRegister src,
+                                         Register tmp1,
+                                         Register tmp2,
+                                         XMMRegister xmm_tmp) {
+  if (!is_reference_type(type)) {
+    BarrierSetAssembler::copy_store_at(masm, decorators, type, bytes, dst, src, tmp1, tmp2, xmm_tmp);
+    return;
+  }
+  Address dst0(dst.base(), dst.index(), dst.scale(), dst.disp() + 0);
+  Address dst1(dst.base(), dst.index(), dst.scale(), dst.disp() + 8);
+  Address dst2(dst.base(), dst.index(), dst.scale(), dst.disp() + 16);
+  Address dst3(dst.base(), dst.index(), dst.scale(), dst.disp() + 24);
+
+  bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
+
+  // Registers set up in the prologue:
+  // xmm2: load_bad_mask
+  // xmm3: store_bad_mask
+  // xmm4: store_good_mask
+
+  if (bytes == 16) {
+    Label done;
+    Label fallback;
+
+    if (UseAVX >= 1) {
+      if (!dest_uninitialized) {
+        // Load destination vector
+        __ movdqu(xmm_tmp, dst);
+        // Check destination store-good
+        __ ptest(xmm_tmp, xmm3);
+        __ jcc(Assembler::notZero, fallback);
+      }
+
+      // Color source
+      __ por(src, xmm4);
+      // Store source in destination
+      __ movdqu(dst, src);
+      __ jmp(done);
+    }
+
+    __ bind(fallback);
+
+    __ subptr(rsp, wordSize * 2);
+    __ movdqu(Address(rsp, 0), src);
+
+    __ movq(tmp1, Address(rsp, 0));
+    ZBarrierSetAssembler::copy_store_at(masm, decorators, type, 8, dst0, tmp1, tmp2);
+    __ movq(tmp1, Address(rsp, 8));
+    ZBarrierSetAssembler::copy_store_at(masm, decorators, type, 8, dst1, tmp1, tmp2);
+
+    __ addptr(rsp, wordSize * 2);
+
+    __ bind(done);
+  } else if (bytes == 32) {
+    Label done;
+    Label fallback;
+    assert(UseAVX >= 2, "Assume UseAVX >= 2");
+
+    if (!dest_uninitialized) {
+      // Load destination vector
+      __ vmovdqu(xmm_tmp, dst);
+      // Check destination store-good
+      __ vptest(xmm_tmp, xmm3, Assembler::AVX_256bit);
+      __ jcc(Assembler::notZero, fallback);
+    }
+
+    // Color source
+    __ vpor(src, src, xmm4, Assembler::AVX_256bit);
+
+    // Store colored source in destination
+    __ vmovdqu(dst, src);
+    __ jmp(done);
+
+    __ bind(fallback);
+
+    __ subptr(rsp, wordSize * 4);
+    __ vmovdqu(Address(rsp, 0), src);
+
+    __ movq(tmp1, Address(rsp, 0));
+    ZBarrierSetAssembler::copy_store_at(masm, decorators, type, 8, dst0, tmp1, tmp2);
+    __ movq(tmp1, Address(rsp, 8));
+    ZBarrierSetAssembler::copy_store_at(masm, decorators, type, 8, dst1, tmp1, tmp2);
+    __ movq(tmp1, Address(rsp, 16));
+    ZBarrierSetAssembler::copy_store_at(masm, decorators, type, 8, dst2, tmp1, tmp2);
+    __ movq(tmp1, Address(rsp, 24));
+    ZBarrierSetAssembler::copy_store_at(masm, decorators, type, 8, dst3, tmp1, tmp2);
+
+    __ addptr(rsp, wordSize * 4);
+
+    __ bind(done);
+  }
+}
 
 void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
                                               DecoratorSet decorators,
@@ -233,14 +900,7 @@ void ZBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm,
 
   BLOCK_COMMENT("ZBarrierSetAssembler::arraycopy_prologue {");
 
-  // Save registers
-  __ pusha();
-
-  // Call VM
-  call_vm(masm, ZBarrierSetRuntime::load_barrier_on_oop_array_addr(), src, count);
-
-  // Restore registers
-  __ popa();
+  load_arraycopy_masks(masm);
 
   BLOCK_COMMENT("} ZBarrierSetAssembler::arraycopy_prologue");
 }
@@ -252,12 +912,51 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
                                                          Label& slowpath) {
   BLOCK_COMMENT("ZBarrierSetAssembler::try_resolve_jobject_in_native {");
 
-  // Resolve jobject
-  BarrierSetAssembler::try_resolve_jobject_in_native(masm, jni_env, obj, tmp, slowpath);
+  Label done, tagged, weak_tagged, uncolor;
 
-  // Test address bad mask
-  __ testptr(obj, address_bad_mask_from_jni_env(jni_env));
+  // Test for tag
+  __ testptr(obj, JNIHandles::tag_mask);
+  __ jcc(Assembler::notZero, tagged);
+
+  // Resolve local handle
+  __ movptr(obj, Address(obj, 0));
+  __ jmp(done);
+
+  __ bind(tagged);
+
+  // Test for weak tag
+  __ testptr(obj, JNIHandles::TypeTag::weak_global);
+  __ jcc(Assembler::notZero, weak_tagged);
+
+  // Resolve global handle
+  __ movptr(obj, Address(obj, -JNIHandles::TypeTag::global));
+  __ testptr(obj, load_bad_mask_from_jni_env(jni_env));
   __ jcc(Assembler::notZero, slowpath);
+  __ jmp(uncolor);
+
+  __ bind(weak_tagged);
+
+  // Resolve weak handle
+  __ movptr(obj, Address(obj, -JNIHandles::TypeTag::weak_global));
+  __ testptr(obj, mark_bad_mask_from_jni_env(jni_env));
+  __ jcc(Assembler::notZero, slowpath);
+
+  __ bind(uncolor);
+
+  // Uncolor
+  if (obj == rcx) {
+    __ movptr(tmp, obj);
+    __ movptr(rcx, ExternalAddress((address)&ZPointerLoadShift));
+    __ shrq(tmp);
+    __ movptr(obj, tmp);
+  } else {
+    __ push(rcx);
+    __ movptr(rcx, ExternalAddress((address)&ZPointerLoadShift));
+    __ shrq(obj);
+    __ pop(rcx);
+  }
+
+  __ bind(done);
 
   BLOCK_COMMENT("} ZBarrierSetAssembler::try_resolve_jobject_in_native");
 }
@@ -267,9 +966,45 @@ void ZBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler* masm,
 #undef __
 #define __ ce->masm()->
 
-void ZBarrierSetAssembler::generate_c1_load_barrier_test(LIR_Assembler* ce,
-                                                         LIR_Opr ref) const {
-  __ testptr(ref->as_register(), address_bad_mask_from_thread(r15_thread));
+static void z_uncolor(LIR_Assembler* ce, LIR_Opr ref) {
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
+  __ shrq(ref->as_register(), barrier_Relocation::unpatched);
+}
+
+static void z_color(LIR_Assembler* ce, LIR_Opr ref) {
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
+  __ shlq(ref->as_register(), barrier_Relocation::unpatched);
+  __ orq_imm32(ref->as_register(), barrier_Relocation::unpatched);
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodAfterOr);
+}
+
+void ZBarrierSetAssembler::generate_c1_uncolor(LIR_Assembler* ce, LIR_Opr ref) const {
+  z_uncolor(ce, ref);
+}
+
+void ZBarrierSetAssembler::generate_c1_color(LIR_Assembler* ce, LIR_Opr ref) const {
+  z_color(ce, ref);
+}
+
+void ZBarrierSetAssembler::generate_c1_load_barrier(LIR_Assembler* ce,
+                                                    LIR_Opr ref,
+                                                    ZLoadBarrierStubC1* stub,
+                                                    bool on_non_strong) const {
+  if (on_non_strong) {
+    // Test against MarkBad mask
+    __ Assembler::testl(ref->as_register(), barrier_Relocation::unpatched);
+    __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatMarkBadAfterTest);
+
+    // Slow path if not zero
+    __ jcc(Assembler::notZero, *stub->entry());
+    // Fast path: convert to colorless
+    z_uncolor(ce, ref);
+  } else {
+    // Convert to colorless and fast path test
+    z_uncolor(ce, ref);
+    __ jcc(Assembler::above, *stub->entry());
+  }
+  __ bind(*stub->continuation());
 }
 
 void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
@@ -281,6 +1016,9 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
   Register ref_addr = noreg;
   Register tmp = noreg;
 
+  // The fast-path shift destroyed the oop - need to re-read it
+  __ movptr(ref, ce->as_Address(stub->ref_addr()->as_address_ptr()));
+
   if (stub->tmp()->is_valid()) {
     // Load address into tmp register
     ce->leal(stub->ref_addr(), stub->tmp());
@@ -321,6 +1059,55 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_stub(LIR_Assembler* ce,
   __ jmp(*stub->continuation());
 }
 
+void ZBarrierSetAssembler::generate_c1_store_barrier(LIR_Assembler* ce,
+                                                     LIR_Address* addr,
+                                                     LIR_Opr new_zaddress,
+                                                     LIR_Opr new_zpointer,
+                                                     ZStoreBarrierStubC1* stub) const {
+  Register rnew_zaddress = new_zaddress->as_register();
+  Register rnew_zpointer = new_zpointer->as_register();
+
+  Register rbase = addr->base()->as_pointer_register();
+  store_barrier_fast(ce->masm(),
+                     ce->as_Address(addr),
+                     rnew_zaddress,
+                     rnew_zpointer,
+                     true,
+                     stub->is_atomic(),
+                     *stub->entry(),
+                     *stub->continuation());
+}
+
+void ZBarrierSetAssembler::generate_c1_store_barrier_stub(LIR_Assembler* ce,
+                                                          ZStoreBarrierStubC1* stub) const {
+  // Stub entry
+  __ bind(*stub->entry());
+
+  Label slow;
+  Label slow_continuation;
+  store_barrier_medium(ce->masm(),
+                       ce->as_Address(stub->ref_addr()->as_address_ptr()),
+                       rscratch1,
+                       false /* is_native */,
+                       stub->is_atomic(),
+                       *stub->continuation(),
+                       slow,
+                       slow_continuation);
+
+  __ bind(slow);
+
+  ce->leal(stub->ref_addr(), stub->new_zpointer());
+
+  // Setup arguments and call runtime stub
+  __ subptr(rsp, 2 * BytesPerWord);
+  ce->store_parameter(stub->new_zpointer()->as_pointer_register(), 0);
+  __ call(RuntimeAddress(stub->runtime_stub()));
+  __ addptr(rsp, 2 * BytesPerWord);
+
+  // Stub exit
+  __ jmp(slow_continuation);
+}
+
 #undef __
 #define __ sasm->
 
@@ -343,6 +1130,28 @@ void ZBarrierSetAssembler::generate_c1_load_barrier_runtime_stub(StubAssembler*
   __ ret(0);
 }
 
+void ZBarrierSetAssembler::generate_c1_store_barrier_runtime_stub(StubAssembler* sasm,
+                                                                  bool self_healing) const {
+  // Enter and save registers
+  __ enter();
+  __ save_live_registers_no_oop_map(true /* save_fpu_registers */);
+
+  // Setup arguments
+  __ load_parameter(0, c_rarg0);
+
+  // Call VM
+  if (self_healing) {
+    __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr(), c_rarg0);
+  } else {
+    __ call_VM_leaf(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr(), c_rarg0);
+  }
+
+  // Restore registers and return
+  __ restore_live_registers(true /* restore_fpu_registers */);
+  __ leave();
+  __ ret(0);
+}
+
 #endif // COMPILER1
 
 #ifdef COMPILER2
@@ -467,7 +1276,7 @@ private:
     _spill_offset += 8;
   }
 
-  void initialize(ZLoadBarrierStubC2* stub) {
+  void initialize(ZBarrierStubC2* stub) {
     // Create mask of caller saved registers that need to
     // be saved/restored if live
     RegMask caller_saved;
@@ -480,13 +1289,13 @@ private:
     caller_saved.Insert(OptoReg::as_OptoReg(r9->as_VMReg()));
     caller_saved.Insert(OptoReg::as_OptoReg(r10->as_VMReg()));
     caller_saved.Insert(OptoReg::as_OptoReg(r11->as_VMReg()));
-    caller_saved.Remove(OptoReg::as_OptoReg(stub->ref()->as_VMReg()));
+
+    if (stub->result() != noreg) {
+      caller_saved.Remove(OptoReg::as_OptoReg(stub->result()->as_VMReg()));
+    }
 
     // Create mask of live registers
     RegMask live = stub->live();
-    if (stub->tmp() != noreg) {
-      live.Insert(OptoReg::as_OptoReg(stub->tmp()->as_VMReg()));
-    }
 
     int gp_spill_size = 0;
     int opmask_spill_size = 0;
@@ -544,7 +1353,7 @@ private:
   }
 
 public:
-  ZSaveLiveRegisters(MacroAssembler* masm, ZLoadBarrierStubC2* stub) :
+  ZSaveLiveRegisters(MacroAssembler* masm, ZBarrierStubC2* stub) :
       _masm(masm),
       _gp_registers(),
       _opmask_registers(),
@@ -683,11 +1492,15 @@ public:
 #define __ masm->
 
 void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, ZLoadBarrierStubC2* stub) const {
+  Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
   BLOCK_COMMENT("ZLoadBarrierStubC2");
 
   // Stub entry
   __ bind(*stub->entry());
 
+  // The fast-path shift destroyed the oop - need to re-read it
+  __ movptr(stub->ref(), stub->ref_addr());
+
   {
     ZSaveLiveRegisters save_live_registers(masm, stub);
     ZSetupArguments setup_arguments(masm, stub);
@@ -698,16 +1511,187 @@ void ZBarrierSetAssembler::generate_c2_load_barrier_stub(MacroAssembler* masm, Z
   __ jmp(*stub->continuation());
 }
 
+void ZBarrierSetAssembler::generate_c2_store_barrier_stub(MacroAssembler* masm, ZStoreBarrierStubC2* stub) const {
+  Assembler::InlineSkippedInstructionsCounter skipped_counter(masm);
+  BLOCK_COMMENT("ZStoreBarrierStubC2");
+
+  // Stub entry
+  __ bind(*stub->entry());
+
+  Label slow;
+  Label slow_continuation;
+  store_barrier_medium(masm,
+                       stub->ref_addr(),
+                       stub->new_zpointer(),
+                       stub->is_native(),
+                       stub->is_atomic(),
+                       *stub->continuation(),
+                       slow,
+                       slow_continuation);
+
+  __ bind(slow);
+
+  {
+    ZSaveLiveRegisters save_live_registers(masm, stub);
+    __ lea(c_rarg0, stub->ref_addr());
+
+    if (stub->is_native()) {
+      __ call(RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr()));
+    } else if (stub->is_atomic()) {
+      __ call(RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr()));
+    } else {
+      __ call(RuntimeAddress(ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr()));
+    }
+  }
+
+  // Stub exit
+  __ jmp(slow_continuation);
+}
+
+#undef __
+
+static int patch_barrier_relocation_offset(int format) {
+  switch (format) {
+  case ZBarrierRelocationFormatLoadGoodBeforeShl:
+    return 3;
+
+  case ZBarrierRelocationFormatStoreGoodAfterCmp:
+    return -2;
+
+  case ZBarrierRelocationFormatLoadBadAfterTest:
+  case ZBarrierRelocationFormatMarkBadAfterTest:
+  case ZBarrierRelocationFormatStoreBadAfterTest:
+  case ZBarrierRelocationFormatStoreGoodAfterOr:
+    return -4;
+  case ZBarrierRelocationFormatStoreGoodAfterMov:
+    return -3;
+
+  default:
+    ShouldNotReachHere();
+    return 0;
+  }
+}
+
+static uint16_t patch_barrier_relocation_value(int format) {
+  switch (format) {
+  case ZBarrierRelocationFormatLoadGoodBeforeShl:
+    return (uint16_t)ZPointerLoadShift;
+
+  case ZBarrierRelocationFormatMarkBadAfterTest:
+    return (uint16_t)ZPointerMarkBadMask;
+
+  case ZBarrierRelocationFormatLoadBadAfterTest:
+    return (uint16_t)ZPointerLoadBadMask;
+
+  case ZBarrierRelocationFormatStoreGoodAfterCmp:
+  case ZBarrierRelocationFormatStoreGoodAfterOr:
+  case ZBarrierRelocationFormatStoreGoodAfterMov:
+    return (uint16_t)ZPointerStoreGoodMask;
+
+  case ZBarrierRelocationFormatStoreBadAfterTest:
+    return (uint16_t)ZPointerStoreBadMask;
+
+  default:
+    ShouldNotReachHere();
+    return 0;
+  }
+}
+
+void ZBarrierSetAssembler::patch_barrier_relocation(address addr, int format) {
+  const int offset = patch_barrier_relocation_offset(format);
+  const uint16_t value = patch_barrier_relocation_value(format);
+  uint8_t* const patch_addr = (uint8_t*)addr + offset;
+  if (format == ZBarrierRelocationFormatLoadGoodBeforeShl) {
+    *patch_addr = (uint8_t)value;
+  } else {
+    *(uint16_t*)patch_addr = value;
+  }
+}
+
+void ZBarrierSetAssembler::patch_barriers() {
+  for (int i = 0; i < _load_bad_relocations.length(); ++i) {
+    address addr = _load_bad_relocations.at(i);
+    patch_barrier_relocation(addr, ZBarrierRelocationFormatLoadBadAfterTest);
+  }
+  for (int i = 0; i < _store_bad_relocations.length(); ++i) {
+    address addr = _store_bad_relocations.at(i);
+    patch_barrier_relocation(addr, ZBarrierRelocationFormatStoreBadAfterTest);
+  }
+  for (int i = 0; i < _store_good_relocations.length(); ++i) {
+    address addr = _store_good_relocations.at(i);
+    patch_barrier_relocation(addr, ZBarrierRelocationFormatStoreGoodAfterOr);
+  }
+}
+
 #endif // COMPILER2
 
 #undef __
 #define __ masm->
 
+
 void ZBarrierSetAssembler::check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error) {
-  // Check if metadata bits indicate a bad oop
-  __ testptr(obj, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset()));
+  // C1 calls verfy_oop in the middle of barriers, before they have been uncolored
+  // and after being colored. Therefore, we must deal with colored oops as well.
+  Label done;
+  Label check_oop;
+  Label check_zaddress;
+  int color_bits = ZPointerRemappedShift + ZPointerRemappedBits;
+
+  uintptr_t shifted_base_start_mask = (UCONST64(1) << (ZAddressHeapBaseShift + color_bits + 1)) - 1;
+  uintptr_t shifted_base_end_mask = (UCONST64(1) << (ZAddressHeapBaseShift + 1)) - 1;
+  uintptr_t shifted_base_mask = shifted_base_start_mask ^ shifted_base_end_mask;
+
+  uintptr_t shifted_address_end_mask = (UCONST64(1) << (color_bits + 1)) - 1;
+  uintptr_t shifted_address_mask = shifted_address_end_mask ^ (uintptr_t)CONST64(-1);
+
+  // Check colored null
+  __ mov64(tmp1, shifted_address_mask);
+  __ testptr(tmp1, obj);
+  __ jcc(Assembler::zero, done);
+
+  // Check for zpointer
+  __ mov64(tmp1, shifted_base_mask);
+  __ testptr(tmp1, obj);
+  __ jcc(Assembler::zero, check_oop);
+
+  // Lookup shift
+  __ movq(tmp1, obj);
+  __ mov64(tmp2, shifted_address_end_mask);
+  __ andq(tmp1, tmp2);
+  __ shrq(tmp1, ZPointerRemappedShift);
+  __ andq(tmp1, (1 << ZPointerRemappedBits) - 1);
+  __ lea(tmp2, ExternalAddress((address)&ZPointerLoadShiftTable));
+
+  // Uncolor presumed zpointer
+  assert(obj != rcx, "bad choice of register");
+  if (rcx != tmp1 && rcx != tmp2) {
+    __ push(rcx);
+  }
+  __ movl(rcx, Address(tmp2, tmp1, Address::times_4, 0));
+  __ shrq(obj);
+  if (rcx != tmp1 && rcx != tmp2) {
+    __ pop(rcx);
+  }
+
+  __ jmp(check_zaddress);
+
+  __ bind(check_oop);
+
+  // make sure klass is 'reasonable', which is not zero.
+  __ load_klass(tmp1, obj, tmp2);  // get klass
+  __ testptr(tmp1, tmp1);
+  __ jcc(Assembler::zero, error); // if klass is null it is broken
+
+  __ bind(check_zaddress);
+  // Check if the oop is in the right area of memory
+  __ movptr(tmp1, obj);
+  __ movptr(tmp2, (intptr_t) Universe::verify_oop_mask());
+  __ andptr(tmp1, tmp2);
+  __ movptr(tmp2, (intptr_t) Universe::verify_oop_bits());
+  __ cmpptr(tmp1, tmp2);
   __ jcc(Assembler::notZero, error);
-  BarrierSetAssembler::check_oop(masm, obj, tmp1, tmp2, error);
+
+  __ bind(done);
 }
 
 #undef __
diff --git a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
index 2c5887361eb..752cff0125d 100644
--- a/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/z/zBarrierSetAssembler_x86.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -33,19 +33,41 @@
 class MacroAssembler;
 
 #ifdef COMPILER1
+class CodeStub;
+class LIR_Address;
 class LIR_Assembler;
 class LIR_Opr;
 class StubAssembler;
 class ZLoadBarrierStubC1;
+class ZStoreBarrierStubC1;
 #endif // COMPILER1
 
 #ifdef COMPILER2
+class MachNode;
 class Node;
 class ZLoadBarrierStubC2;
+class ZStoreBarrierStubC2;
 #endif // COMPILER2
 
+const int ZBarrierRelocationFormatLoadGoodBeforeShl = 0;
+const int ZBarrierRelocationFormatLoadBadAfterTest  = 1;
+const int ZBarrierRelocationFormatMarkBadAfterTest  = 2;
+const int ZBarrierRelocationFormatStoreGoodAfterCmp = 3;
+const int ZBarrierRelocationFormatStoreBadAfterTest = 4;
+const int ZBarrierRelocationFormatStoreGoodAfterOr  = 5;
+const int ZBarrierRelocationFormatStoreGoodAfterMov = 6;
+
 class ZBarrierSetAssembler : public ZBarrierSetAssemblerBase {
+private:
+  GrowableArrayCHeap<address, mtGC> _load_bad_relocations;
+  GrowableArrayCHeap<address, mtGC> _store_bad_relocations;
+  GrowableArrayCHeap<address, mtGC> _store_good_relocations;
+
 public:
+  static const int32_t _zpointer_address_mask = 0xFFFF0000;
+
+  ZBarrierSetAssembler();
+
   virtual void load_at(MacroAssembler* masm,
                        DecoratorSet decorators,
                        BasicType type,
@@ -54,7 +76,6 @@ public:
                        Register tmp1,
                        Register tmp_thread);
 
-#ifdef ASSERT
   virtual void store_at(MacroAssembler* masm,
                         DecoratorSet decorators,
                         BasicType type,
@@ -63,7 +84,43 @@ public:
                         Register tmp1,
                         Register tmp2,
                         Register tmp3);
-#endif // ASSERT
+
+  virtual bool supports_avx3_masked_arraycopy();
+
+  virtual void copy_load_at(MacroAssembler* masm,
+                            DecoratorSet decorators,
+                            BasicType type,
+                            size_t bytes,
+                            Register dst,
+                            Address src,
+                            Register tmp);
+
+  virtual void copy_store_at(MacroAssembler* masm,
+                             DecoratorSet decorators,
+                             BasicType type,
+                             size_t bytes,
+                             Address dst,
+                             Register src,
+                             Register tmp);
+
+  virtual void copy_load_at(MacroAssembler* masm,
+                            DecoratorSet decorators,
+                            BasicType type,
+                            size_t bytes,
+                            XMMRegister dst,
+                            Address src,
+                            Register tmp,
+                            XMMRegister xmm_tmp);
+
+  virtual void copy_store_at(MacroAssembler* masm,
+                             DecoratorSet decorators,
+                             BasicType type,
+                             size_t bytes,
+                             Address dst,
+                             XMMRegister src,
+                             Register tmp1,
+                             Register tmp2,
+                             XMMRegister xmm_tmp);
 
   virtual void arraycopy_prologue(MacroAssembler* masm,
                                   DecoratorSet decorators,
@@ -79,8 +136,25 @@ public:
                                              Label& slowpath);
 
 #ifdef COMPILER1
-  void generate_c1_load_barrier_test(LIR_Assembler* ce,
-                                     LIR_Opr ref) const;
+  void generate_c1_color(LIR_Assembler* ce, LIR_Opr ref) const;
+  void generate_c1_uncolor(LIR_Assembler* ce, LIR_Opr ref) const;
+
+  void generate_c1_store_barrier(LIR_Assembler* ce,
+                                 LIR_Address* addr,
+                                 LIR_Opr new_zaddress,
+                                 LIR_Opr new_zpointer,
+                                 ZStoreBarrierStubC1* stub) const;
+
+  void generate_c1_store_barrier_stub(LIR_Assembler* ce,
+                                      ZStoreBarrierStubC1* stub) const;
+
+  void generate_c1_store_barrier_runtime_stub(StubAssembler* sasm,
+                                              bool self_healing) const;
+
+  void generate_c1_load_barrier(LIR_Assembler* ce,
+                                LIR_Opr ref,
+                                ZLoadBarrierStubC1* stub,
+                                bool on_non_strong) const;
 
   void generate_c1_load_barrier_stub(LIR_Assembler* ce,
                                      ZLoadBarrierStubC1* stub) const;
@@ -95,8 +169,32 @@ public:
 
   void generate_c2_load_barrier_stub(MacroAssembler* masm,
                                      ZLoadBarrierStubC2* stub) const;
+  void generate_c2_store_barrier_stub(MacroAssembler* masm,
+                                      ZStoreBarrierStubC2* stub) const;
 #endif // COMPILER2
 
+  void store_barrier_fast(MacroAssembler* masm,
+                          Address ref_addr,
+                          Register rnew_persistent,
+                          Register rnew_transient,
+                          bool in_nmethod,
+                          bool is_atomic,
+                          Label& medium_path,
+                          Label& medium_path_continuation) const;
+
+  void store_barrier_medium(MacroAssembler* masm,
+                            Address ref_addr,
+                            Register tmp,
+                            bool is_native,
+                            bool is_atomic,
+                            Label& medium_path_continuation,
+                            Label& slow_path,
+                            Label& slow_path_continuation) const;
+
+  void patch_barrier_relocation(address addr, int format);
+
+  void patch_barriers();
+
   void check_oop(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2, Label& error);
 };
 
diff --git a/src/hotspot/cpu/x86/gc/z/zGlobals_x86.hpp b/src/hotspot/cpu/x86/gc/z/zGlobals_x86.hpp
index 51b29b2a32c..5ee5aef9baf 100644
--- a/src/hotspot/cpu/x86/gc/z/zGlobals_x86.hpp
+++ b/src/hotspot/cpu/x86/gc/z/zGlobals_x86.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,10 +24,6 @@
 #ifndef CPU_X86_GC_Z_ZGLOBALS_X86_HPP
 #define CPU_X86_GC_Z_ZGLOBALS_X86_HPP
 
-const size_t ZPlatformHeapViews        = 3;
 const size_t ZPlatformCacheLineSize    = 64;
 
-size_t ZPlatformAddressOffsetBits();
-size_t ZPlatformAddressMetadataShift();
-
 #endif // CPU_X86_GC_Z_ZGLOBALS_X86_HPP
diff --git a/src/hotspot/cpu/x86/gc/z/z_x86_64.ad b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad
index 44f3f221fae..adb0f3ab8eb 100644
--- a/src/hotspot/cpu/x86/gc/z/z_x86_64.ad
+++ b/src/hotspot/cpu/x86/gc/z/z_x86_64.ad
@@ -1,5 +1,5 @@
 //
-// Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
@@ -32,52 +32,66 @@ source_hpp %{
 source %{
 
 #include "c2_intelJccErratum_x86.hpp"
+#include "gc/z/zBarrierSetAssembler.hpp"
 
-static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data) {
-  if (barrier_data == ZLoadBarrierElided) {
+static void z_color(MacroAssembler& _masm, const MachNode* node, Register ref) {
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
+  __ shlq(ref, barrier_Relocation::unpatched);
+  __ orq_imm32(ref, barrier_Relocation::unpatched);
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodAfterOr);
+}
+
+static void z_uncolor(MacroAssembler& _masm, const MachNode* node, Register ref) {
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatLoadGoodBeforeShl);
+  __ shrq(ref, barrier_Relocation::unpatched);
+}
+
+static void z_keep_alive_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref) {
+  __ Assembler::testl(ref, barrier_Relocation::unpatched);
+  __ relocate(barrier_Relocation::spec(), ZBarrierRelocationFormatMarkBadAfterTest);
+
+  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref);
+  __ jcc(Assembler::notEqual, *stub->entry());
+
+  z_uncolor(_masm, node, ref);
+
+  __ bind(*stub->continuation());
+}
+
+static void z_load_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref) {
+  Assembler::InlineSkippedInstructionsCounter skipped_counter(&_masm);
+  const bool on_non_strong =
+      ((node->barrier_data() & ZBarrierWeak) != 0) ||
+      ((node->barrier_data() & ZBarrierPhantom) != 0);
+
+  if (on_non_strong) {
+    z_keep_alive_load_barrier(_masm, node, ref_addr, ref);
     return;
   }
-  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, barrier_data);
-  {
-    IntelJccErratumAlignment intel_alignment(_masm, 10 /* jcc_size */);
-    __ testptr(ref, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset()));
-    __ jcc(Assembler::notZero, *stub->entry());
+
+  z_uncolor(_masm, node, ref);
+  if (node->barrier_data() == ZBarrierElided) {
+    return;
   }
+  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref);
+  IntelJccErratumAlignment(_masm, 6);
+  __ jcc(Assembler::above, *stub->entry());
   __ bind(*stub->continuation());
 }
 
-static void z_load_barrier_cmpxchg(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register ref, Register tmp, Label& good) {
-  ZLoadBarrierStubC2* const stub = ZLoadBarrierStubC2::create(node, ref_addr, ref, tmp, ZLoadBarrierStrong);
-  {
-    IntelJccErratumAlignment intel_alignment(_masm, 10 /* jcc_size */);
-    __ testptr(ref, Address(r15_thread, ZThreadLocalData::address_bad_mask_offset()));
-    __ jcc(Assembler::zero, good);
-  }
-  {
-    IntelJccErratumAlignment intel_alignment(_masm, 5 /* jcc_size */);
-    __ jmp(*stub->entry());
-  }
-  __ bind(*stub->continuation());
-}
-
-static void z_cmpxchg_common(MacroAssembler& _masm, const MachNode* node, Register mem_reg, Register newval, Register tmp) {
-  // Compare value (oldval) is in rax
-   const Address mem = Address(mem_reg, 0);
-
-  if (node->barrier_data() != ZLoadBarrierElided) {
-    __ movptr(tmp, rax);
-  }
-
-  __ lock();
-  __ cmpxchgptr(newval, mem);
-
-  if (node->barrier_data() != ZLoadBarrierElided) {
-    Label good;
-    z_load_barrier_cmpxchg(_masm, node, mem, rax, tmp, good);
-    __ movptr(rax, tmp);
-    __ lock();
-    __ cmpxchgptr(newval, mem);
-    __ bind(good);
+static void z_store_barrier(MacroAssembler& _masm, const MachNode* node, Address ref_addr, Register rnew_zaddress, Register rnew_zpointer, bool is_atomic) {
+  Assembler::InlineSkippedInstructionsCounter skipped_counter(&_masm);
+  if (node->barrier_data() == ZBarrierElided) {
+    if (rnew_zaddress != noreg) {
+      // noreg means null; no need to color
+      __ movptr(rnew_zpointer, rnew_zaddress);
+      z_color(_masm, node, rnew_zpointer);
+    }
+  } else {
+    bool is_native = (node->barrier_data() & ZBarrierNative) != 0;
+    ZStoreBarrierStubC2* const stub = ZStoreBarrierStubC2::create(node, ref_addr, rnew_zaddress, rnew_zpointer, is_native, is_atomic);
+    ZBarrierSetAssembler* bs_asm = ZBarrierSet::assembler();
+    bs_asm->store_barrier_fast(&_masm, ref_addr, rnew_zaddress, rnew_zpointer, true /* in_nmethod */, is_atomic, *stub->entry(), *stub->continuation());
   }
 }
 
@@ -86,9 +100,9 @@ static void z_cmpxchg_common(MacroAssembler& _masm, const MachNode* node, Regist
 // Load Pointer
 instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr)
 %{
-  predicate(UseZGC && n->as_Load()->barrier_data() != 0);
+  predicate(UseZGC && ZGenerational && n->as_Load()->barrier_data() != 0);
   match(Set dst (LoadP mem));
-  effect(KILL cr, TEMP dst);
+  effect(TEMP dst, KILL cr);
 
   ins_cost(125);
 
@@ -96,33 +110,91 @@ instruct zLoadP(rRegP dst, memory mem, rFlagsReg cr)
 
   ins_encode %{
     __ movptr($dst$$Register, $mem$$Address);
-    z_load_barrier(_masm, this, $mem$$Address, $dst$$Register, noreg /* tmp */, barrier_data());
+    z_load_barrier(_masm, this, $mem$$Address, $dst$$Register);
   %}
 
   ins_pipe(ialu_reg_mem);
 %}
 
-instruct zCompareAndExchangeP(indirect mem, rax_RegP oldval, rRegP newval, rRegP tmp, rFlagsReg cr) %{
+// Load Pointer and Null Check
+instruct zLoadPNullCheck(rFlagsReg cr, memory op, immP0 zero)
+%{
+  predicate(UseZGC && ZGenerational && n->in(1)->as_Load()->barrier_data() != 0);
+  match(Set cr (CmpP (LoadP op) zero));
+
+  ins_cost(500); // XXX
+  format %{ "testq   $op, 0xffffffffffff0000\t# ptr" %}
+  ins_encode %{
+    // A null pointer will have all address bits 0. This mask sign extends
+    // all address bits, so we can test if the address is 0.
+    __ testq($op$$Address, ZBarrierSetAssembler::_zpointer_address_mask);
+  %}
+  ins_pipe(ialu_cr_reg_imm);
+%}
+
+// Store Pointer
+instruct zStoreP(memory mem, any_RegP src, rRegP tmp, rFlagsReg cr)
+%{
+  predicate(UseZGC && ZGenerational && n->as_Store()->barrier_data() != 0);
+  match(Set mem (StoreP mem src));
+  effect(TEMP tmp, KILL cr);
+
+  ins_cost(125); // XXX
+  format %{ "movq    $mem, $src\t# ptr" %}
+  ins_encode %{
+    z_store_barrier(_masm, this, $mem$$Address, $src$$Register, $tmp$$Register, false /* is_atomic */);
+    __ movq($mem$$Address, $tmp$$Register);
+  %}
+  ins_pipe(ialu_mem_reg);
+%}
+
+// Store Null Pointer
+instruct zStorePNull(memory mem, immP0 zero, rRegP tmp, rFlagsReg cr)
+%{
+  predicate(UseZGC && ZGenerational && n->as_Store()->barrier_data() != 0);
+  match(Set mem (StoreP mem zero));
+  effect(TEMP tmp, KILL cr);
+
+  ins_cost(125); // XXX
+  format %{ "movq    $mem, 0\t# ptr" %}
+  ins_encode %{
+    z_store_barrier(_masm, this, $mem$$Address, noreg, $tmp$$Register, false /* is_atomic */);
+    // Store a colored null - barrier code above does not need to color
+    __ movq($mem$$Address, barrier_Relocation::unpatched);
+    // The relocation cant be fully after the mov, as that is the beginning of a random subsequent
+    // instruction, which violates assumptions made by unrelated code. Hence the end() - 1
+    __ code_section()->relocate(__ code_section()->end() - 1, barrier_Relocation::spec(), ZBarrierRelocationFormatStoreGoodAfterMov);
+  %}
+  ins_pipe(ialu_mem_reg);
+%}
+
+instruct zCompareAndExchangeP(indirect mem, no_rax_RegP newval, rRegP tmp, rax_RegP oldval, rFlagsReg cr) %{
   match(Set oldval (CompareAndExchangeP mem (Binary oldval newval)));
-  predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
-  effect(KILL cr, TEMP tmp);
+  predicate(UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP tmp, KILL cr);
 
   format %{ "lock\n\t"
             "cmpxchgq $newval, $mem" %}
 
   ins_encode %{
-    precond($oldval$$Register == rax);
-    z_cmpxchg_common(_masm, this, $mem$$Register, $newval$$Register, $tmp$$Register);
+    assert_different_registers($oldval$$Register, $mem$$Register);
+    assert_different_registers($oldval$$Register, $newval$$Register);
+    const Address mem_addr = Address($mem$$Register, 0);
+    z_store_barrier(_masm, this, mem_addr, $newval$$Register, $tmp$$Register, true /* is_atomic */);
+    z_color(_masm, this, $oldval$$Register);
+    __ lock();
+    __ cmpxchgptr($tmp$$Register, mem_addr);
+    z_uncolor(_masm, this, $oldval$$Register);
   %}
 
   ins_pipe(pipe_cmpxchg);
 %}
 
-instruct zCompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp, rFlagsReg cr, rax_RegP oldval) %{
+instruct zCompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp, rax_RegP oldval, rFlagsReg cr) %{
   match(Set res (CompareAndSwapP mem (Binary oldval newval)));
   match(Set res (WeakCompareAndSwapP mem (Binary oldval newval)));
-  predicate(UseZGC && n->as_LoadStore()->barrier_data() == ZLoadBarrierStrong);
-  effect(KILL cr, KILL oldval, TEMP tmp);
+  predicate(UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP tmp, KILL oldval, KILL cr);
 
   format %{ "lock\n\t"
             "cmpxchgq $newval, $mem\n\t"
@@ -130,11 +202,12 @@ instruct zCompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp, rFla
             "movzbl   $res, $res" %}
 
   ins_encode %{
-    precond($oldval$$Register == rax);
-    z_cmpxchg_common(_masm, this, $mem$$Register, $newval$$Register, $tmp$$Register);
-    if (barrier_data() != ZLoadBarrierElided) {
-      __ cmpptr($tmp$$Register, rax);
-    }
+    assert_different_registers($oldval$$Register, $mem$$Register);
+    const Address mem_addr = Address($mem$$Register, 0);
+    z_store_barrier(_masm, this, mem_addr, $newval$$Register, $tmp$$Register, true /* is_atomic */);
+    z_color(_masm, this, $oldval$$Register);
+    __ lock();
+    __ cmpxchgptr($tmp$$Register, mem_addr);
     __ setb(Assembler::equal, $res$$Register);
     __ movzbl($res$$Register, $res$$Register);
   %}
@@ -142,16 +215,20 @@ instruct zCompareAndSwapP(rRegI res, indirect mem, rRegP newval, rRegP tmp, rFla
   ins_pipe(pipe_cmpxchg);
 %}
 
-instruct zXChgP(indirect mem, rRegP newval, rFlagsReg cr) %{
+instruct zXChgP(indirect mem, rRegP newval, rRegP tmp, rFlagsReg cr) %{
   match(Set newval (GetAndSetP mem newval));
-  predicate(UseZGC && n->as_LoadStore()->barrier_data() != 0);
-  effect(KILL cr);
+  predicate(UseZGC && ZGenerational && n->as_LoadStore()->barrier_data() != 0);
+  effect(TEMP tmp, KILL cr);
 
   format %{ "xchgq    $newval, $mem" %}
 
   ins_encode %{
-    __ xchgptr($newval$$Register, Address($mem$$Register, 0));
-    z_load_barrier(_masm, this, Address(noreg, 0), $newval$$Register, noreg /* tmp */, barrier_data());
+    assert_different_registers($mem$$Register, $newval$$Register);
+    const Address mem_addr = Address($mem$$Register, 0);
+    z_store_barrier(_masm, this, mem_addr, $newval$$Register, $tmp$$Register, true /* is_atomic */);
+    __ movptr($newval$$Register, $tmp$$Register);
+    __ xchgptr($newval$$Register, mem_addr);
+    z_uncolor(_masm, this, $newval$$Register);
   %}
 
   ins_pipe(pipe_cmpxchg);
diff --git a/src/hotspot/cpu/x86/relocInfo_x86.hpp b/src/hotspot/cpu/x86/relocInfo_x86.hpp
index 621bf7ae728..d3f213a6686 100644
--- a/src/hotspot/cpu/x86/relocInfo_x86.hpp
+++ b/src/hotspot/cpu/x86/relocInfo_x86.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -35,8 +35,8 @@
 #ifndef AMD64
     format_width       =  1
 #else
-    // vs Assembler::narrow_oop_operand.
-    format_width       =  2
+    // vs Assembler::narrow_oop_operand and ZGC barrier encodings.
+    format_width       =  3
 #endif
   };
 
diff --git a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
index 489ea3c69cc..6cd17651514 100644
--- a/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
+++ b/src/hotspot/cpu/x86/stubGenerator_x86_64.cpp
@@ -44,9 +44,6 @@
 #if INCLUDE_JVMCI
 #include "jvmci/jvmci_globals.hpp"
 #endif
-#if INCLUDE_ZGC
-#include "gc/z/zThreadLocalData.hpp"
-#endif
 #if INCLUDE_JFR
 #include "jfr/support/jfrIntrinsics.hpp"
 #endif
diff --git a/src/hotspot/cpu/x86/stubRoutines_x86.hpp b/src/hotspot/cpu/x86/stubRoutines_x86.hpp
index 3f94520b664..2038fdff5ae 100644
--- a/src/hotspot/cpu/x86/stubRoutines_x86.hpp
+++ b/src/hotspot/cpu/x86/stubRoutines_x86.hpp
@@ -38,7 +38,7 @@ enum platform_dependent_constants {
   // AVX512 intrinsics add more code in 64-bit VM,
   // Windows have more code to save/restore registers
   _compiler_stubs_code_size     = 20000 LP64_ONLY(+30000) WINDOWS_ONLY(+2000),
-  _final_stubs_code_size        = 10000 LP64_ONLY(+20000) WINDOWS_ONLY(+2000)
+  _final_stubs_code_size        = 10000 LP64_ONLY(+20000) WINDOWS_ONLY(+2000) ZGC_ONLY(+20000)
 };
 
 class x86 {
diff --git a/src/hotspot/cpu/x86/templateTable_x86.cpp b/src/hotspot/cpu/x86/templateTable_x86.cpp
index 6144e10e8e2..15efa6ed21d 100644
--- a/src/hotspot/cpu/x86/templateTable_x86.cpp
+++ b/src/hotspot/cpu/x86/templateTable_x86.cpp
@@ -26,6 +26,7 @@
 #include "asm/macroAssembler.hpp"
 #include "compiler/disassembler.hpp"
 #include "gc/shared/collectedHeap.hpp"
+#include "gc/shared/gc_globals.hpp"
 #include "gc/shared/tlab_globals.hpp"
 #include "interpreter/interpreter.hpp"
 #include "interpreter/interpreterRuntime.hpp"
diff --git a/src/hotspot/os/bsd/gc/x/xLargePages_bsd.cpp b/src/hotspot/os/bsd/gc/x/xLargePages_bsd.cpp
new file mode 100644
index 00000000000..1c82e831208
--- /dev/null
+++ b/src/hotspot/os/bsd/gc/x/xLargePages_bsd.cpp
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xLargePages.hpp"
+#include "runtime/globals.hpp"
+
+void XLargePages::pd_initialize() {
+  if (UseLargePages) {
+    _state = Explicit;
+  } else {
+    _state = Disabled;
+  }
+}
diff --git a/src/hotspot/os/bsd/gc/x/xNUMA_bsd.cpp b/src/hotspot/os/bsd/gc/x/xNUMA_bsd.cpp
new file mode 100644
index 00000000000..b0e23a1716a
--- /dev/null
+++ b/src/hotspot/os/bsd/gc/x/xNUMA_bsd.cpp
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xNUMA.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+void XNUMA::pd_initialize() {
+  _enabled = false;
+}
+
+uint32_t XNUMA::count() {
+  return 1;
+}
+
+uint32_t XNUMA::id() {
+  return 0;
+}
+
+uint32_t XNUMA::memory_id(uintptr_t addr) {
+  // NUMA support not enabled, assume everything belongs to node zero
+  return 0;
+}
diff --git a/src/hotspot/os/bsd/gc/x/xPhysicalMemoryBacking_bsd.cpp b/src/hotspot/os/bsd/gc/x/xPhysicalMemoryBacking_bsd.cpp
new file mode 100644
index 00000000000..2c64c3788d3
--- /dev/null
+++ b/src/hotspot/os/bsd/gc/x/xPhysicalMemoryBacking_bsd.cpp
@@ -0,0 +1,181 @@
+/*
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/x/xErrno.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xLargePages.inline.hpp"
+#include "gc/x/xPhysicalMemory.inline.hpp"
+#include "gc/x/xPhysicalMemoryBacking_bsd.hpp"
+#include "logging/log.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/os.hpp"
+#include "utilities/align.hpp"
+#include "utilities/debug.hpp"
+
+#include <mach/mach.h>
+#include <mach/mach_vm.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+
+// The backing is represented by a reserved virtual address space, in which
+// we commit and uncommit physical memory. Multi-mapping the different heap
+// views is done by simply remapping the backing memory using mach_vm_remap().
+
+static int vm_flags_superpage() {
+  if (!XLargePages::is_explicit()) {
+    return 0;
+  }
+
+  const int page_size_in_megabytes = XGranuleSize >> 20;
+  return page_size_in_megabytes << VM_FLAGS_SUPERPAGE_SHIFT;
+}
+
+static XErrno mremap(uintptr_t from_addr, uintptr_t to_addr, size_t size) {
+  mach_vm_address_t remap_addr = to_addr;
+  vm_prot_t remap_cur_prot;
+  vm_prot_t remap_max_prot;
+
+  // Remap memory to an additional location
+  const kern_return_t res = mach_vm_remap(mach_task_self(),
+                                          &remap_addr,
+                                          size,
+                                          0 /* mask */,
+                                          VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE | vm_flags_superpage(),
+                                          mach_task_self(),
+                                          from_addr,
+                                          FALSE /* copy */,
+                                          &remap_cur_prot,
+                                          &remap_max_prot,
+                                          VM_INHERIT_COPY);
+
+  return (res == KERN_SUCCESS) ? XErrno(0) : XErrno(EINVAL);
+}
+
+XPhysicalMemoryBacking::XPhysicalMemoryBacking(size_t max_capacity) :
+    _base(0),
+    _initialized(false) {
+
+  // Reserve address space for backing memory
+  _base = (uintptr_t)os::reserve_memory(max_capacity);
+  if (_base == 0) {
+    // Failed
+    log_error_pd(gc)("Failed to reserve address space for backing memory");
+    return;
+  }
+
+  // Successfully initialized
+  _initialized = true;
+}
+
+bool XPhysicalMemoryBacking::is_initialized() const {
+  return _initialized;
+}
+
+void XPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const {
+  // Does nothing
+}
+
+bool XPhysicalMemoryBacking::commit_inner(size_t offset, size_t length) const {
+  assert(is_aligned(offset, os::vm_page_size()), "Invalid offset");
+  assert(is_aligned(length, os::vm_page_size()), "Invalid length");
+
+  log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
+                      offset / M, (offset + length) / M, length / M);
+
+  const uintptr_t addr = _base + offset;
+  const void* const res = mmap((void*)addr, length, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  if (res == MAP_FAILED) {
+    XErrno err;
+    log_error(gc)("Failed to commit memory (%s)", err.to_string());
+    return false;
+  }
+
+  // Success
+  return true;
+}
+
+size_t XPhysicalMemoryBacking::commit(size_t offset, size_t length) const {
+  // Try to commit the whole region
+  if (commit_inner(offset, length)) {
+    // Success
+    return length;
+  }
+
+  // Failed, try to commit as much as possible
+  size_t start = offset;
+  size_t end = offset + length;
+
+  for (;;) {
+    length = align_down((end - start) / 2, XGranuleSize);
+    if (length == 0) {
+      // Done, don't commit more
+      return start - offset;
+    }
+
+    if (commit_inner(start, length)) {
+      // Success, try commit more
+      start += length;
+    } else {
+      // Failed, try commit less
+      end -= length;
+    }
+  }
+}
+
+size_t XPhysicalMemoryBacking::uncommit(size_t offset, size_t length) const {
+  assert(is_aligned(offset, os::vm_page_size()), "Invalid offset");
+  assert(is_aligned(length, os::vm_page_size()), "Invalid length");
+
+  log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
+                      offset / M, (offset + length) / M, length / M);
+
+  const uintptr_t start = _base + offset;
+  const void* const res = mmap((void*)start, length, PROT_NONE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
+  if (res == MAP_FAILED) {
+    XErrno err;
+    log_error(gc)("Failed to uncommit memory (%s)", err.to_string());
+    return 0;
+  }
+
+  return length;
+}
+
+void XPhysicalMemoryBacking::map(uintptr_t addr, size_t size, uintptr_t offset) const {
+  const XErrno err = mremap(_base + offset, addr, size);
+  if (err) {
+    fatal("Failed to remap memory (%s)", err.to_string());
+  }
+}
+
+void XPhysicalMemoryBacking::unmap(uintptr_t addr, size_t size) const {
+  // Note that we must keep the address space reservation intact and just detach
+  // the backing memory. For this reason we map a new anonymous, non-accessible
+  // and non-reserved page over the mapping instead of actually unmapping.
+  const void* const res = mmap((void*)addr, size, PROT_NONE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
+  if (res == MAP_FAILED) {
+    XErrno err;
+    fatal("Failed to map memory (%s)", err.to_string());
+  }
+}
diff --git a/src/hotspot/os/bsd/gc/x/xPhysicalMemoryBacking_bsd.hpp b/src/hotspot/os/bsd/gc/x/xPhysicalMemoryBacking_bsd.hpp
new file mode 100644
index 00000000000..8b4747026ff
--- /dev/null
+++ b/src/hotspot/os/bsd/gc/x/xPhysicalMemoryBacking_bsd.hpp
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef OS_BSD_GC_X_XPHYSICALMEMORYBACKING_BSD_HPP
+#define OS_BSD_GC_X_XPHYSICALMEMORYBACKING_BSD_HPP
+
+class XPhysicalMemoryBacking {
+private:
+  uintptr_t _base;
+  bool      _initialized;
+
+  bool commit_inner(size_t offset, size_t length) const;
+
+public:
+  XPhysicalMemoryBacking(size_t max_capacity);
+
+  bool is_initialized() const;
+
+  void warn_commit_limits(size_t max_capacity) const;
+
+  size_t commit(size_t offset, size_t length) const;
+  size_t uncommit(size_t offset, size_t length) const;
+
+  void map(uintptr_t addr, size_t size, uintptr_t offset) const;
+  void unmap(uintptr_t addr, size_t size) const;
+};
+
+#endif // OS_BSD_GC_X_XPHYSICALMEMORYBACKING_BSD_HPP
diff --git a/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.cpp b/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.cpp
index ea9a54bcb39..218173a739a 100644
--- a/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.cpp
+++ b/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,7 @@
 
 #include "precompiled.hpp"
 #include "gc/shared/gcLogPrecious.hpp"
+#include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zErrno.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zLargePages.inline.hpp"
@@ -97,14 +98,14 @@ void ZPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const {
   // Does nothing
 }
 
-bool ZPhysicalMemoryBacking::commit_inner(size_t offset, size_t length) const {
-  assert(is_aligned(offset, os::vm_page_size()), "Invalid offset");
+bool ZPhysicalMemoryBacking::commit_inner(zoffset offset, size_t length) const {
+  assert(is_aligned(untype(offset), os::vm_page_size()), "Invalid offset");
   assert(is_aligned(length, os::vm_page_size()), "Invalid length");
 
   log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
-                      offset / M, (offset + length) / M, length / M);
+                      untype(offset) / M, untype(offset) + length / M, length / M);
 
-  const uintptr_t addr = _base + offset;
+  const uintptr_t addr = _base + untype(offset);
   const void* const res = mmap((void*)addr, length, PROT_READ | PROT_WRITE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
   if (res == MAP_FAILED) {
     ZErrno err;
@@ -116,7 +117,7 @@ bool ZPhysicalMemoryBacking::commit_inner(size_t offset, size_t length) const {
   return true;
 }
 
-size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) const {
+size_t ZPhysicalMemoryBacking::commit(zoffset offset, size_t length) const {
   // Try to commit the whole region
   if (commit_inner(offset, length)) {
     // Success
@@ -124,8 +125,8 @@ size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) const {
   }
 
   // Failed, try to commit as much as possible
-  size_t start = offset;
-  size_t end = offset + length;
+  zoffset start = offset;
+  zoffset end = offset + length;
 
   for (;;) {
     length = align_down((end - start) / 2, ZGranuleSize);
@@ -144,14 +145,14 @@ size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) const {
   }
 }
 
-size_t ZPhysicalMemoryBacking::uncommit(size_t offset, size_t length) const {
-  assert(is_aligned(offset, os::vm_page_size()), "Invalid offset");
+size_t ZPhysicalMemoryBacking::uncommit(zoffset offset, size_t length) const {
+  assert(is_aligned(untype(offset), os::vm_page_size()), "Invalid offset");
   assert(is_aligned(length, os::vm_page_size()), "Invalid length");
 
   log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
-                      offset / M, (offset + length) / M, length / M);
+                      untype(offset) / M, untype(offset) + length / M, length / M);
 
-  const uintptr_t start = _base + offset;
+  const uintptr_t start = _base + untype(offset);
   const void* const res = mmap((void*)start, length, PROT_NONE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
   if (res == MAP_FAILED) {
     ZErrno err;
@@ -162,18 +163,18 @@ size_t ZPhysicalMemoryBacking::uncommit(size_t offset, size_t length) const {
   return length;
 }
 
-void ZPhysicalMemoryBacking::map(uintptr_t addr, size_t size, uintptr_t offset) const {
-  const ZErrno err = mremap(_base + offset, addr, size);
+void ZPhysicalMemoryBacking::map(zaddress_unsafe addr, size_t size, zoffset offset) const {
+  const ZErrno err = mremap(_base + untype(offset), untype(addr), size);
   if (err) {
     fatal("Failed to remap memory (%s)", err.to_string());
   }
 }
 
-void ZPhysicalMemoryBacking::unmap(uintptr_t addr, size_t size) const {
+void ZPhysicalMemoryBacking::unmap(zaddress_unsafe addr, size_t size) const {
   // Note that we must keep the address space reservation intact and just detach
   // the backing memory. For this reason we map a new anonymous, non-accessible
   // and non-reserved page over the mapping instead of actually unmapping.
-  const void* const res = mmap((void*)addr, size, PROT_NONE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
+  const void* const res = mmap((void*)untype(addr), size, PROT_NONE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
   if (res == MAP_FAILED) {
     ZErrno err;
     fatal("Failed to map memory (%s)", err.to_string());
diff --git a/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.hpp b/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.hpp
index ca61ec65eea..d74de5375ee 100644
--- a/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.hpp
+++ b/src/hotspot/os/bsd/gc/z/zPhysicalMemoryBacking_bsd.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,12 +24,14 @@
 #ifndef OS_BSD_GC_Z_ZPHYSICALMEMORYBACKING_BSD_HPP
 #define OS_BSD_GC_Z_ZPHYSICALMEMORYBACKING_BSD_HPP
 
+#include "gc/z/zAddress.hpp"
+
 class ZPhysicalMemoryBacking {
 private:
   uintptr_t _base;
   bool      _initialized;
 
-  bool commit_inner(size_t offset, size_t length) const;
+  bool commit_inner(zoffset offset, size_t length) const;
 
 public:
   ZPhysicalMemoryBacking(size_t max_capacity);
@@ -38,11 +40,11 @@ public:
 
   void warn_commit_limits(size_t max_capacity) const;
 
-  size_t commit(size_t offset, size_t length) const;
-  size_t uncommit(size_t offset, size_t length) const;
+  size_t commit(zoffset offset, size_t length) const;
+  size_t uncommit(zoffset offset, size_t length) const;
 
-  void map(uintptr_t addr, size_t size, uintptr_t offset) const;
-  void unmap(uintptr_t addr, size_t size) const;
+  void map(zaddress_unsafe addr, size_t size, zoffset offset) const;
+  void unmap(zaddress_unsafe addr, size_t size) const;
 };
 
 #endif // OS_BSD_GC_Z_ZPHYSICALMEMORYBACKING_BSD_HPP
diff --git a/src/hotspot/os/linux/gc/x/xLargePages_linux.cpp b/src/hotspot/os/linux/gc/x/xLargePages_linux.cpp
new file mode 100644
index 00000000000..6ad956b1e63
--- /dev/null
+++ b/src/hotspot/os/linux/gc/x/xLargePages_linux.cpp
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xLargePages.hpp"
+#include "runtime/globals.hpp"
+
+void XLargePages::pd_initialize() {
+  if (UseLargePages) {
+    if (UseTransparentHugePages) {
+      _state = Transparent;
+    } else {
+      _state = Explicit;
+    }
+  } else {
+    _state = Disabled;
+  }
+}
diff --git a/src/hotspot/os/linux/gc/x/xMountPoint_linux.cpp b/src/hotspot/os/linux/gc/x/xMountPoint_linux.cpp
new file mode 100644
index 00000000000..96c0f2f92db
--- /dev/null
+++ b/src/hotspot/os/linux/gc/x/xMountPoint_linux.cpp
@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/x/xArray.inline.hpp"
+#include "gc/x/xErrno.hpp"
+#include "gc/x/xMountPoint_linux.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/os.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+#include <stdio.h>
+#include <unistd.h>
+
+// Mount information, see proc(5) for more details.
+#define PROC_SELF_MOUNTINFO        "/proc/self/mountinfo"
+
+XMountPoint::XMountPoint(const char* filesystem, const char** preferred_mountpoints) {
+  if (AllocateHeapAt != nullptr) {
+    // Use specified path
+    _path = os::strdup(AllocateHeapAt, mtGC);
+  } else {
+    // Find suitable path
+    _path = find_mountpoint(filesystem, preferred_mountpoints);
+  }
+}
+
+XMountPoint::~XMountPoint() {
+  os::free(_path);
+  _path = nullptr;
+}
+
+char* XMountPoint::get_mountpoint(const char* line, const char* filesystem) const {
+  char* line_mountpoint = nullptr;
+  char* line_filesystem = nullptr;
+
+  // Parse line and return a newly allocated string containing the mount point if
+  // the line contains a matching filesystem and the mount point is accessible by
+  // the current user.
+  // sscanf, using %m, will return malloced memory. Need raw ::free, not os::free.
+  if (sscanf(line, "%*u %*u %*u:%*u %*s %ms %*[^-]- %ms", &line_mountpoint, &line_filesystem) != 2 ||
+      strcmp(line_filesystem, filesystem) != 0 ||
+      access(line_mountpoint, R_OK|W_OK|X_OK) != 0) {
+    // Not a matching or accessible filesystem
+    ALLOW_C_FUNCTION(::free, ::free(line_mountpoint);)
+    line_mountpoint = nullptr;
+  }
+
+  ALLOW_C_FUNCTION(::free, ::free(line_filesystem);)
+
+  return line_mountpoint;
+}
+
+void XMountPoint::get_mountpoints(const char* filesystem, XArray<char*>* mountpoints) const {
+  FILE* fd = os::fopen(PROC_SELF_MOUNTINFO, "r");
+  if (fd == nullptr) {
+    XErrno err;
+    log_error_p(gc)("Failed to open %s: %s", PROC_SELF_MOUNTINFO, err.to_string());
+    return;
+  }
+
+  char* line = nullptr;
+  size_t length = 0;
+
+  while (getline(&line, &length, fd) != -1) {
+    char* const mountpoint = get_mountpoint(line, filesystem);
+    if (mountpoint != nullptr) {
+      mountpoints->append(mountpoint);
+    }
+  }
+
+  // readline will return malloced memory. Need raw ::free, not os::free.
+  ALLOW_C_FUNCTION(::free, ::free(line);)
+  fclose(fd);
+}
+
+void XMountPoint::free_mountpoints(XArray<char*>* mountpoints) const {
+  XArrayIterator<char*> iter(mountpoints);
+  for (char* mountpoint; iter.next(&mountpoint);) {
+    ALLOW_C_FUNCTION(::free, ::free(mountpoint);) // *not* os::free
+  }
+  mountpoints->clear();
+}
+
+char* XMountPoint::find_preferred_mountpoint(const char* filesystem,
+                                              XArray<char*>* mountpoints,
+                                              const char** preferred_mountpoints) const {
+  // Find preferred mount point
+  XArrayIterator<char*> iter1(mountpoints);
+  for (char* mountpoint; iter1.next(&mountpoint);) {
+    for (const char** preferred = preferred_mountpoints; *preferred != nullptr; preferred++) {
+      if (!strcmp(mountpoint, *preferred)) {
+        // Preferred mount point found
+        return os::strdup(mountpoint, mtGC);
+      }
+    }
+  }
+
+  // Preferred mount point not found
+  log_error_p(gc)("More than one %s filesystem found:", filesystem);
+  XArrayIterator<char*> iter2(mountpoints);
+  for (char* mountpoint; iter2.next(&mountpoint);) {
+    log_error_p(gc)("  %s", mountpoint);
+  }
+
+  return nullptr;
+}
+
+char* XMountPoint::find_mountpoint(const char* filesystem, const char** preferred_mountpoints) const {
+  char* path = nullptr;
+  XArray<char*> mountpoints;
+
+  get_mountpoints(filesystem, &mountpoints);
+
+  if (mountpoints.length() == 0) {
+    // No mount point found
+    log_error_p(gc)("Failed to find an accessible %s filesystem", filesystem);
+  } else if (mountpoints.length() == 1) {
+    // One mount point found
+    path = os::strdup(mountpoints.at(0), mtGC);
+  } else {
+    // More than one mount point found
+    path = find_preferred_mountpoint(filesystem, &mountpoints, preferred_mountpoints);
+  }
+
+  free_mountpoints(&mountpoints);
+
+  return path;
+}
+
+const char* XMountPoint::get() const {
+  return _path;
+}
diff --git a/src/hotspot/os/linux/gc/x/xMountPoint_linux.hpp b/src/hotspot/os/linux/gc/x/xMountPoint_linux.hpp
new file mode 100644
index 00000000000..e0ca126e066
--- /dev/null
+++ b/src/hotspot/os/linux/gc/x/xMountPoint_linux.hpp
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef OS_LINUX_GC_X_XMOUNTPOINT_LINUX_HPP
+#define OS_LINUX_GC_X_XMOUNTPOINT_LINUX_HPP
+
+#include "gc/x/xArray.hpp"
+#include "memory/allocation.hpp"
+
+class XMountPoint : public StackObj {
+private:
+  char* _path;
+
+  char* get_mountpoint(const char* line,
+                       const char* filesystem) const;
+  void get_mountpoints(const char* filesystem,
+                       XArray<char*>* mountpoints) const;
+  void free_mountpoints(XArray<char*>* mountpoints) const;
+  char* find_preferred_mountpoint(const char* filesystem,
+                                  XArray<char*>* mountpoints,
+                                  const char** preferred_mountpoints) const;
+  char* find_mountpoint(const char* filesystem,
+                        const char** preferred_mountpoints) const;
+
+public:
+  XMountPoint(const char* filesystem, const char** preferred_mountpoints);
+  ~XMountPoint();
+
+  const char* get() const;
+};
+
+#endif // OS_LINUX_GC_X_XMOUNTPOINT_LINUX_HPP
diff --git a/src/hotspot/os/linux/gc/x/xNUMA_linux.cpp b/src/hotspot/os/linux/gc/x/xNUMA_linux.cpp
new file mode 100644
index 00000000000..0cc557dde6e
--- /dev/null
+++ b/src/hotspot/os/linux/gc/x/xNUMA_linux.cpp
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "gc/x/xCPU.inline.hpp"
+#include "gc/x/xErrno.hpp"
+#include "gc/x/xNUMA.hpp"
+#include "gc/x/xSyscall_linux.hpp"
+#include "os_linux.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/os.hpp"
+#include "utilities/debug.hpp"
+
+void XNUMA::pd_initialize() {
+  _enabled = UseNUMA;
+}
+
+uint32_t XNUMA::count() {
+  if (!_enabled) {
+    // NUMA support not enabled
+    return 1;
+  }
+
+  return os::Linux::numa_max_node() + 1;
+}
+
+uint32_t XNUMA::id() {
+  if (!_enabled) {
+    // NUMA support not enabled
+    return 0;
+  }
+
+  return os::Linux::get_node_by_cpu(XCPU::id());
+}
+
+uint32_t XNUMA::memory_id(uintptr_t addr) {
+  if (!_enabled) {
+    // NUMA support not enabled, assume everything belongs to node zero
+    return 0;
+  }
+
+  uint32_t id = (uint32_t)-1;
+
+  if (XSyscall::get_mempolicy((int*)&id, nullptr, 0, (void*)addr, MPOL_F_NODE | MPOL_F_ADDR) == -1) {
+    XErrno err;
+    fatal("Failed to get NUMA id for memory at " PTR_FORMAT " (%s)", addr, err.to_string());
+  }
+
+  assert(id < count(), "Invalid NUMA id");
+
+  return id;
+}
diff --git a/src/hotspot/os/linux/gc/x/xPhysicalMemoryBacking_linux.cpp b/src/hotspot/os/linux/gc/x/xPhysicalMemoryBacking_linux.cpp
new file mode 100644
index 00000000000..5db59741a58
--- /dev/null
+++ b/src/hotspot/os/linux/gc/x/xPhysicalMemoryBacking_linux.cpp
@@ -0,0 +1,724 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/x/xArray.inline.hpp"
+#include "gc/x/xErrno.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xLargePages.inline.hpp"
+#include "gc/x/xMountPoint_linux.hpp"
+#include "gc/x/xNUMA.inline.hpp"
+#include "gc/x/xPhysicalMemoryBacking_linux.hpp"
+#include "gc/x/xSyscall_linux.hpp"
+#include "logging/log.hpp"
+#include "os_linux.hpp"
+#include "runtime/init.hpp"
+#include "runtime/os.hpp"
+#include "runtime/safefetch.hpp"
+#include "utilities/align.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/growableArray.hpp"
+
+#include <fcntl.h>
+#include <stdio.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/statfs.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+//
+// Support for building on older Linux systems
+//
+
+// memfd_create(2) flags
+#ifndef MFD_CLOEXEC
+#define MFD_CLOEXEC                      0x0001U
+#endif
+#ifndef MFD_HUGETLB
+#define MFD_HUGETLB                      0x0004U
+#endif
+#ifndef MFD_HUGE_2MB
+#define MFD_HUGE_2MB                     0x54000000U
+#endif
+
+// open(2) flags
+#ifndef O_CLOEXEC
+#define O_CLOEXEC                        02000000
+#endif
+#ifndef O_TMPFILE
+#define O_TMPFILE                        (020000000 | O_DIRECTORY)
+#endif
+
+// fallocate(2) flags
+#ifndef FALLOC_FL_KEEP_SIZE
+#define FALLOC_FL_KEEP_SIZE              0x01
+#endif
+#ifndef FALLOC_FL_PUNCH_HOLE
+#define FALLOC_FL_PUNCH_HOLE             0x02
+#endif
+
+// Filesystem types, see statfs(2)
+#ifndef TMPFS_MAGIC
+#define TMPFS_MAGIC                      0x01021994
+#endif
+#ifndef HUGETLBFS_MAGIC
+#define HUGETLBFS_MAGIC                  0x958458f6
+#endif
+
+// Filesystem names
+#define XFILESYSTEM_TMPFS                "tmpfs"
+#define XFILESYSTEM_HUGETLBFS            "hugetlbfs"
+
+// Proc file entry for max map mount
+#define XFILENAME_PROC_MAX_MAP_COUNT     "/proc/sys/vm/max_map_count"
+
+// Sysfs file for transparent huge page on tmpfs
+#define XFILENAME_SHMEM_ENABLED          "/sys/kernel/mm/transparent_hugepage/shmem_enabled"
+
+// Java heap filename
+#define XFILENAME_HEAP                   "java_heap"
+
+// Preferred tmpfs mount points, ordered by priority
+static const char* z_preferred_tmpfs_mountpoints[] = {
+  "/dev/shm",
+  "/run/shm",
+  nullptr
+};
+
+// Preferred hugetlbfs mount points, ordered by priority
+static const char* z_preferred_hugetlbfs_mountpoints[] = {
+  "/dev/hugepages",
+  "/hugepages",
+  nullptr
+};
+
+static int z_fallocate_hugetlbfs_attempts = 3;
+static bool z_fallocate_supported = true;
+
+XPhysicalMemoryBacking::XPhysicalMemoryBacking(size_t max_capacity) :
+    _fd(-1),
+    _filesystem(0),
+    _block_size(0),
+    _available(0),
+    _initialized(false) {
+
+  // Create backing file
+  _fd = create_fd(XFILENAME_HEAP);
+  if (_fd == -1) {
+    return;
+  }
+
+  // Truncate backing file
+  while (ftruncate(_fd, max_capacity) == -1) {
+    if (errno != EINTR) {
+      XErrno err;
+      log_error_p(gc)("Failed to truncate backing file (%s)", err.to_string());
+      return;
+    }
+  }
+
+  // Get filesystem statistics
+  struct statfs buf;
+  if (fstatfs(_fd, &buf) == -1) {
+    XErrno err;
+    log_error_p(gc)("Failed to determine filesystem type for backing file (%s)", err.to_string());
+    return;
+  }
+
+  _filesystem = buf.f_type;
+  _block_size = buf.f_bsize;
+  _available = buf.f_bavail * _block_size;
+
+  log_info_p(gc, init)("Heap Backing Filesystem: %s (" UINT64_FORMAT_X ")",
+                       is_tmpfs() ? XFILESYSTEM_TMPFS : is_hugetlbfs() ? XFILESYSTEM_HUGETLBFS : "other", _filesystem);
+
+  // Make sure the filesystem type matches requested large page type
+  if (XLargePages::is_transparent() && !is_tmpfs()) {
+    log_error_p(gc)("-XX:+UseTransparentHugePages can only be enabled when using a %s filesystem",
+                    XFILESYSTEM_TMPFS);
+    return;
+  }
+
+  if (XLargePages::is_transparent() && !tmpfs_supports_transparent_huge_pages()) {
+    log_error_p(gc)("-XX:+UseTransparentHugePages on a %s filesystem not supported by kernel",
+                    XFILESYSTEM_TMPFS);
+    return;
+  }
+
+  if (XLargePages::is_explicit() && !is_hugetlbfs()) {
+    log_error_p(gc)("-XX:+UseLargePages (without -XX:+UseTransparentHugePages) can only be enabled "
+                    "when using a %s filesystem", XFILESYSTEM_HUGETLBFS);
+    return;
+  }
+
+  if (!XLargePages::is_explicit() && is_hugetlbfs()) {
+    log_error_p(gc)("-XX:+UseLargePages must be enabled when using a %s filesystem",
+                    XFILESYSTEM_HUGETLBFS);
+    return;
+  }
+
+  // Make sure the filesystem block size is compatible
+  if (XGranuleSize % _block_size != 0) {
+    log_error_p(gc)("Filesystem backing the heap has incompatible block size (" SIZE_FORMAT ")",
+                    _block_size);
+    return;
+  }
+
+  if (is_hugetlbfs() && _block_size != XGranuleSize) {
+    log_error_p(gc)("%s filesystem has unexpected block size " SIZE_FORMAT " (expected " SIZE_FORMAT ")",
+                    XFILESYSTEM_HUGETLBFS, _block_size, XGranuleSize);
+    return;
+  }
+
+  // Successfully initialized
+  _initialized = true;
+}
+
+int XPhysicalMemoryBacking::create_mem_fd(const char* name) const {
+  assert(XGranuleSize == 2 * M, "Granule size must match MFD_HUGE_2MB");
+
+  // Create file name
+  char filename[PATH_MAX];
+  snprintf(filename, sizeof(filename), "%s%s", name, XLargePages::is_explicit() ? ".hugetlb" : "");
+
+  // Create file
+  const int extra_flags = XLargePages::is_explicit() ? (MFD_HUGETLB | MFD_HUGE_2MB) : 0;
+  const int fd = XSyscall::memfd_create(filename, MFD_CLOEXEC | extra_flags);
+  if (fd == -1) {
+    XErrno err;
+    log_debug_p(gc, init)("Failed to create memfd file (%s)",
+                          (XLargePages::is_explicit() && (err == EINVAL || err == ENODEV)) ?
+                          "Hugepages (2M) not available" : err.to_string());
+    return -1;
+  }
+
+  log_info_p(gc, init)("Heap Backing File: /memfd:%s", filename);
+
+  return fd;
+}
+
+int XPhysicalMemoryBacking::create_file_fd(const char* name) const {
+  const char* const filesystem = XLargePages::is_explicit()
+                                 ? XFILESYSTEM_HUGETLBFS
+                                 : XFILESYSTEM_TMPFS;
+  const char** const preferred_mountpoints = XLargePages::is_explicit()
+                                             ? z_preferred_hugetlbfs_mountpoints
+                                             : z_preferred_tmpfs_mountpoints;
+
+  // Find mountpoint
+  XMountPoint mountpoint(filesystem, preferred_mountpoints);
+  if (mountpoint.get() == nullptr) {
+    log_error_p(gc)("Use -XX:AllocateHeapAt to specify the path to a %s filesystem", filesystem);
+    return -1;
+  }
+
+  // Try to create an anonymous file using the O_TMPFILE flag. Note that this
+  // flag requires kernel >= 3.11. If this fails we fall back to open/unlink.
+  const int fd_anon = os::open(mountpoint.get(), O_TMPFILE|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR);
+  if (fd_anon == -1) {
+    XErrno err;
+    log_debug_p(gc, init)("Failed to create anonymous file in %s (%s)", mountpoint.get(),
+                          (err == EINVAL ? "Not supported" : err.to_string()));
+  } else {
+    // Get inode number for anonymous file
+    struct stat stat_buf;
+    if (fstat(fd_anon, &stat_buf) == -1) {
+      XErrno err;
+      log_error_pd(gc)("Failed to determine inode number for anonymous file (%s)", err.to_string());
+      return -1;
+    }
+
+    log_info_p(gc, init)("Heap Backing File: %s/#" UINT64_FORMAT, mountpoint.get(), (uint64_t)stat_buf.st_ino);
+
+    return fd_anon;
+  }
+
+  log_debug_p(gc, init)("Falling back to open/unlink");
+
+  // Create file name
+  char filename[PATH_MAX];
+  snprintf(filename, sizeof(filename), "%s/%s.%d", mountpoint.get(), name, os::current_process_id());
+
+  // Create file
+  const int fd = os::open(filename, O_CREAT|O_EXCL|O_RDWR|O_CLOEXEC, S_IRUSR|S_IWUSR);
+  if (fd == -1) {
+    XErrno err;
+    log_error_p(gc)("Failed to create file %s (%s)", filename, err.to_string());
+    return -1;
+  }
+
+  // Unlink file
+  if (unlink(filename) == -1) {
+    XErrno err;
+    log_error_p(gc)("Failed to unlink file %s (%s)", filename, err.to_string());
+    return -1;
+  }
+
+  log_info_p(gc, init)("Heap Backing File: %s", filename);
+
+  return fd;
+}
+
+int XPhysicalMemoryBacking::create_fd(const char* name) const {
+  if (AllocateHeapAt == nullptr) {
+    // If the path is not explicitly specified, then we first try to create a memfd file
+    // instead of looking for a tmpfd/hugetlbfs mount point. Note that memfd_create() might
+    // not be supported at all (requires kernel >= 3.17), or it might not support large
+    // pages (requires kernel >= 4.14). If memfd_create() fails, then we try to create a
+    // file on an accessible tmpfs or hugetlbfs mount point.
+    const int fd = create_mem_fd(name);
+    if (fd != -1) {
+      return fd;
+    }
+
+    log_debug_p(gc)("Falling back to searching for an accessible mount point");
+  }
+
+  return create_file_fd(name);
+}
+
+bool XPhysicalMemoryBacking::is_initialized() const {
+  return _initialized;
+}
+
+void XPhysicalMemoryBacking::warn_available_space(size_t max_capacity) const {
+  // Note that the available space on a tmpfs or a hugetlbfs filesystem
+  // will be zero if no size limit was specified when it was mounted.
+  if (_available == 0) {
+    // No size limit set, skip check
+    log_info_p(gc, init)("Available space on backing filesystem: N/A");
+    return;
+  }
+
+  log_info_p(gc, init)("Available space on backing filesystem: " SIZE_FORMAT "M", _available / M);
+
+  // Warn if the filesystem doesn't currently have enough space available to hold
+  // the max heap size. The max heap size will be capped if we later hit this limit
+  // when trying to expand the heap.
+  if (_available < max_capacity) {
+    log_warning_p(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****");
+    log_warning_p(gc)("Not enough space available on the backing filesystem to hold the current max Java heap");
+    log_warning_p(gc)("size (" SIZE_FORMAT "M). Please adjust the size of the backing filesystem accordingly "
+                      "(available", max_capacity / M);
+    log_warning_p(gc)("space is currently " SIZE_FORMAT "M). Continuing execution with the current filesystem "
+                      "size could", _available / M);
+    log_warning_p(gc)("lead to a premature OutOfMemoryError being thrown, due to failure to commit memory.");
+  }
+}
+
+void XPhysicalMemoryBacking::warn_max_map_count(size_t max_capacity) const {
+  const char* const filename = XFILENAME_PROC_MAX_MAP_COUNT;
+  FILE* const file = os::fopen(filename, "r");
+  if (file == nullptr) {
+    // Failed to open file, skip check
+    log_debug_p(gc, init)("Failed to open %s", filename);
+    return;
+  }
+
+  size_t actual_max_map_count = 0;
+  const int result = fscanf(file, SIZE_FORMAT, &actual_max_map_count);
+  fclose(file);
+  if (result != 1) {
+    // Failed to read file, skip check
+    log_debug_p(gc, init)("Failed to read %s", filename);
+    return;
+  }
+
+  // The required max map count is impossible to calculate exactly since subsystems
+  // other than ZGC are also creating memory mappings, and we have no control over that.
+  // However, ZGC tends to create the most mappings and dominate the total count.
+  // In the worst cases, ZGC will map each granule three times, i.e. once per heap view.
+  // We speculate that we need another 20% to allow for non-ZGC subsystems to map memory.
+  const size_t required_max_map_count = (max_capacity / XGranuleSize) * 3 * 1.2;
+  if (actual_max_map_count < required_max_map_count) {
+    log_warning_p(gc)("***** WARNING! INCORRECT SYSTEM CONFIGURATION DETECTED! *****");
+    log_warning_p(gc)("The system limit on number of memory mappings per process might be too low for the given");
+    log_warning_p(gc)("max Java heap size (" SIZE_FORMAT "M). Please adjust %s to allow for at",
+                      max_capacity / M, filename);
+    log_warning_p(gc)("least " SIZE_FORMAT " mappings (current limit is " SIZE_FORMAT "). Continuing execution "
+                      "with the current", required_max_map_count, actual_max_map_count);
+    log_warning_p(gc)("limit could lead to a premature OutOfMemoryError being thrown, due to failure to map memory.");
+  }
+}
+
+void XPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const {
+  // Warn if available space is too low
+  warn_available_space(max_capacity);
+
+  // Warn if max map count is too low
+  warn_max_map_count(max_capacity);
+}
+
+bool XPhysicalMemoryBacking::is_tmpfs() const {
+  return _filesystem == TMPFS_MAGIC;
+}
+
+bool XPhysicalMemoryBacking::is_hugetlbfs() const {
+  return _filesystem == HUGETLBFS_MAGIC;
+}
+
+bool XPhysicalMemoryBacking::tmpfs_supports_transparent_huge_pages() const {
+  // If the shmem_enabled file exists and is readable then we
+  // know the kernel supports transparent huge pages for tmpfs.
+  return access(XFILENAME_SHMEM_ENABLED, R_OK) == 0;
+}
+
+XErrno XPhysicalMemoryBacking::fallocate_compat_mmap_hugetlbfs(size_t offset, size_t length, bool touch) const {
+  // On hugetlbfs, mapping a file segment will fail immediately, without
+  // the need to touch the mapped pages first, if there aren't enough huge
+  // pages available to back the mapping.
+  void* const addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset);
+  if (addr == MAP_FAILED) {
+    // Failed
+    return errno;
+  }
+
+  // Once mapped, the huge pages are only reserved. We need to touch them
+  // to associate them with the file segment. Note that we can not punch
+  // hole in file segments which only have reserved pages.
+  if (touch) {
+    char* const start = (char*)addr;
+    char* const end = start + length;
+    os::pretouch_memory(start, end, _block_size);
+  }
+
+  // Unmap again. From now on, the huge pages that were mapped are allocated
+  // to this file. There's no risk of getting a SIGBUS when mapping and
+  // touching these pages again.
+  if (munmap(addr, length) == -1) {
+    // Failed
+    return errno;
+  }
+
+  // Success
+  return 0;
+}
+
+static bool safe_touch_mapping(void* addr, size_t length, size_t page_size) {
+  char* const start = (char*)addr;
+  char* const end = start + length;
+
+  // Touching a mapping that can't be backed by memory will generate a
+  // SIGBUS. By using SafeFetch32 any SIGBUS will be safely caught and
+  // handled. On tmpfs, doing a fetch (rather than a store) is enough
+  // to cause backing pages to be allocated (there's no zero-page to
+  // worry about).
+  for (char *p = start; p < end; p += page_size) {
+    if (SafeFetch32((int*)p, -1) == -1) {
+      // Failed
+      return false;
+    }
+  }
+
+  // Success
+  return true;
+}
+
+XErrno XPhysicalMemoryBacking::fallocate_compat_mmap_tmpfs(size_t offset, size_t length) const {
+  // On tmpfs, we need to touch the mapped pages to figure out
+  // if there are enough pages available to back the mapping.
+  void* const addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset);
+  if (addr == MAP_FAILED) {
+    // Failed
+    return errno;
+  }
+
+  // Advise mapping to use transparent huge pages
+  os::realign_memory((char*)addr, length, XGranuleSize);
+
+  // Touch the mapping (safely) to make sure it's backed by memory
+  const bool backed = safe_touch_mapping(addr, length, _block_size);
+
+  // Unmap again. If successfully touched, the backing memory will
+  // be allocated to this file. There's no risk of getting a SIGBUS
+  // when mapping and touching these pages again.
+  if (munmap(addr, length) == -1) {
+    // Failed
+    return errno;
+  }
+
+  // Success
+  return backed ? 0 : ENOMEM;
+}
+
+XErrno XPhysicalMemoryBacking::fallocate_compat_pwrite(size_t offset, size_t length) const {
+  uint8_t data = 0;
+
+  // Allocate backing memory by writing to each block
+  for (size_t pos = offset; pos < offset + length; pos += _block_size) {
+    if (pwrite(_fd, &data, sizeof(data), pos) == -1) {
+      // Failed
+      return errno;
+    }
+  }
+
+  // Success
+  return 0;
+}
+
+XErrno XPhysicalMemoryBacking::fallocate_fill_hole_compat(size_t offset, size_t length) const {
+  // fallocate(2) is only supported by tmpfs since Linux 3.5, and by hugetlbfs
+  // since Linux 4.3. When fallocate(2) is not supported we emulate it using
+  // mmap/munmap (for hugetlbfs and tmpfs with transparent huge pages) or pwrite
+  // (for tmpfs without transparent huge pages and other filesystem types).
+  if (XLargePages::is_explicit()) {
+    return fallocate_compat_mmap_hugetlbfs(offset, length, false /* touch */);
+  } else if (XLargePages::is_transparent()) {
+    return fallocate_compat_mmap_tmpfs(offset, length);
+  } else {
+    return fallocate_compat_pwrite(offset, length);
+  }
+}
+
+XErrno XPhysicalMemoryBacking::fallocate_fill_hole_syscall(size_t offset, size_t length) const {
+  const int mode = 0; // Allocate
+  const int res = XSyscall::fallocate(_fd, mode, offset, length);
+  if (res == -1) {
+    // Failed
+    return errno;
+  }
+
+  // Success
+  return 0;
+}
+
+XErrno XPhysicalMemoryBacking::fallocate_fill_hole(size_t offset, size_t length) const {
+  // Using compat mode is more efficient when allocating space on hugetlbfs.
+  // Note that allocating huge pages this way will only reserve them, and not
+  // associate them with segments of the file. We must guarantee that we at
+  // some point touch these segments, otherwise we can not punch hole in them.
+  // Also note that we need to use compat mode when using transparent huge pages,
+  // since we need to use madvise(2) on the mapping before the page is allocated.
+  if (z_fallocate_supported && !XLargePages::is_enabled()) {
+     const XErrno err = fallocate_fill_hole_syscall(offset, length);
+     if (!err) {
+       // Success
+       return 0;
+     }
+
+     if (err != ENOSYS && err != EOPNOTSUPP) {
+       // Failed
+       return err;
+     }
+
+     // Not supported
+     log_debug_p(gc)("Falling back to fallocate() compatibility mode");
+     z_fallocate_supported = false;
+  }
+
+  return fallocate_fill_hole_compat(offset, length);
+}
+
+XErrno XPhysicalMemoryBacking::fallocate_punch_hole(size_t offset, size_t length) const {
+  if (XLargePages::is_explicit()) {
+    // We can only punch hole in pages that have been touched. Non-touched
+    // pages are only reserved, and not associated with any specific file
+    // segment. We don't know which pages have been previously touched, so
+    // we always touch them here to guarantee that we can punch hole.
+    const XErrno err = fallocate_compat_mmap_hugetlbfs(offset, length, true /* touch */);
+    if (err) {
+      // Failed
+      return err;
+    }
+  }
+
+  const int mode = FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE;
+  if (XSyscall::fallocate(_fd, mode, offset, length) == -1) {
+    // Failed
+    return errno;
+  }
+
+  // Success
+  return 0;
+}
+
+XErrno XPhysicalMemoryBacking::split_and_fallocate(bool punch_hole, size_t offset, size_t length) const {
+  // Try first half
+  const size_t offset0 = offset;
+  const size_t length0 = align_up(length / 2, _block_size);
+  const XErrno err0 = fallocate(punch_hole, offset0, length0);
+  if (err0) {
+    return err0;
+  }
+
+  // Try second half
+  const size_t offset1 = offset0 + length0;
+  const size_t length1 = length - length0;
+  const XErrno err1 = fallocate(punch_hole, offset1, length1);
+  if (err1) {
+    return err1;
+  }
+
+  // Success
+  return 0;
+}
+
+XErrno XPhysicalMemoryBacking::fallocate(bool punch_hole, size_t offset, size_t length) const {
+  assert(is_aligned(offset, _block_size), "Invalid offset");
+  assert(is_aligned(length, _block_size), "Invalid length");
+
+  const XErrno err = punch_hole ? fallocate_punch_hole(offset, length) : fallocate_fill_hole(offset, length);
+  if (err == EINTR && length > _block_size) {
+    // Calling fallocate(2) with a large length can take a long time to
+    // complete. When running profilers, such as VTune, this syscall will
+    // be constantly interrupted by signals. Expanding the file in smaller
+    // steps avoids this problem.
+    return split_and_fallocate(punch_hole, offset, length);
+  }
+
+  return err;
+}
+
+bool XPhysicalMemoryBacking::commit_inner(size_t offset, size_t length) const {
+  log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
+                      offset / M, (offset + length) / M, length / M);
+
+retry:
+  const XErrno err = fallocate(false /* punch_hole */, offset, length);
+  if (err) {
+    if (err == ENOSPC && !is_init_completed() && XLargePages::is_explicit() && z_fallocate_hugetlbfs_attempts-- > 0) {
+      // If we fail to allocate during initialization, due to lack of space on
+      // the hugetlbfs filesystem, then we wait and retry a few times before
+      // giving up. Otherwise there is a risk that running JVMs back-to-back
+      // will fail, since there is a delay between process termination and the
+      // huge pages owned by that process being returned to the huge page pool
+      // and made available for new allocations.
+      log_debug_p(gc, init)("Failed to commit memory (%s), retrying", err.to_string());
+
+      // Wait and retry in one second, in the hope that huge pages will be
+      // available by then.
+      sleep(1);
+      goto retry;
+    }
+
+    // Failed
+    log_error_p(gc)("Failed to commit memory (%s)", err.to_string());
+    return false;
+  }
+
+  // Success
+  return true;
+}
+
+static int offset_to_node(size_t offset) {
+  const GrowableArray<int>* mapping = os::Linux::numa_nindex_to_node();
+  const size_t nindex = (offset >> XGranuleSizeShift) % mapping->length();
+  return mapping->at((int)nindex);
+}
+
+size_t XPhysicalMemoryBacking::commit_numa_interleaved(size_t offset, size_t length) const {
+  size_t committed = 0;
+
+  // Commit one granule at a time, so that each granule
+  // can be allocated from a different preferred node.
+  while (committed < length) {
+    const size_t granule_offset = offset + committed;
+
+    // Setup NUMA policy to allocate memory from a preferred node
+    os::Linux::numa_set_preferred(offset_to_node(granule_offset));
+
+    if (!commit_inner(granule_offset, XGranuleSize)) {
+      // Failed
+      break;
+    }
+
+    committed += XGranuleSize;
+  }
+
+  // Restore NUMA policy
+  os::Linux::numa_set_preferred(-1);
+
+  return committed;
+}
+
+size_t XPhysicalMemoryBacking::commit_default(size_t offset, size_t length) const {
+  // Try to commit the whole region
+  if (commit_inner(offset, length)) {
+    // Success
+    return length;
+  }
+
+  // Failed, try to commit as much as possible
+  size_t start = offset;
+  size_t end = offset + length;
+
+  for (;;) {
+    length = align_down((end - start) / 2, XGranuleSize);
+    if (length < XGranuleSize) {
+      // Done, don't commit more
+      return start - offset;
+    }
+
+    if (commit_inner(start, length)) {
+      // Success, try commit more
+      start += length;
+    } else {
+      // Failed, try commit less
+      end -= length;
+    }
+  }
+}
+
+size_t XPhysicalMemoryBacking::commit(size_t offset, size_t length) const {
+  if (XNUMA::is_enabled() && !XLargePages::is_explicit()) {
+    // To get granule-level NUMA interleaving when using non-large pages,
+    // we must explicitly interleave the memory at commit/fallocate time.
+    return commit_numa_interleaved(offset, length);
+  }
+
+  return commit_default(offset, length);
+}
+
+size_t XPhysicalMemoryBacking::uncommit(size_t offset, size_t length) const {
+  log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
+                      offset / M, (offset + length) / M, length / M);
+
+  const XErrno err = fallocate(true /* punch_hole */, offset, length);
+  if (err) {
+    log_error(gc)("Failed to uncommit memory (%s)", err.to_string());
+    return 0;
+  }
+
+  return length;
+}
+
+void XPhysicalMemoryBacking::map(uintptr_t addr, size_t size, uintptr_t offset) const {
+  const void* const res = mmap((void*)addr, size, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _fd, offset);
+  if (res == MAP_FAILED) {
+    XErrno err;
+    fatal("Failed to map memory (%s)", err.to_string());
+  }
+}
+
+void XPhysicalMemoryBacking::unmap(uintptr_t addr, size_t size) const {
+  // Note that we must keep the address space reservation intact and just detach
+  // the backing memory. For this reason we map a new anonymous, non-accessible
+  // and non-reserved page over the mapping instead of actually unmapping.
+  const void* const res = mmap((void*)addr, size, PROT_NONE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
+  if (res == MAP_FAILED) {
+    XErrno err;
+    fatal("Failed to map memory (%s)", err.to_string());
+  }
+}
diff --git a/src/hotspot/os/linux/gc/x/xPhysicalMemoryBacking_linux.hpp b/src/hotspot/os/linux/gc/x/xPhysicalMemoryBacking_linux.hpp
new file mode 100644
index 00000000000..253a3f87ef4
--- /dev/null
+++ b/src/hotspot/os/linux/gc/x/xPhysicalMemoryBacking_linux.hpp
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef OS_LINUX_GC_X_XPHYSICALMEMORYBACKING_LINUX_HPP
+#define OS_LINUX_GC_X_XPHYSICALMEMORYBACKING_LINUX_HPP
+
+class XErrno;
+
+class XPhysicalMemoryBacking {
+private:
+  int      _fd;
+  size_t   _size;
+  uint64_t _filesystem;
+  size_t   _block_size;
+  size_t   _available;
+  bool     _initialized;
+
+  void warn_available_space(size_t max_capacity) const;
+  void warn_max_map_count(size_t max_capacity) const;
+
+  int create_mem_fd(const char* name) const;
+  int create_file_fd(const char* name) const;
+  int create_fd(const char* name) const;
+
+  bool is_tmpfs() const;
+  bool is_hugetlbfs() const;
+  bool tmpfs_supports_transparent_huge_pages() const;
+
+  XErrno fallocate_compat_mmap_hugetlbfs(size_t offset, size_t length, bool touch) const;
+  XErrno fallocate_compat_mmap_tmpfs(size_t offset, size_t length) const;
+  XErrno fallocate_compat_pwrite(size_t offset, size_t length) const;
+  XErrno fallocate_fill_hole_compat(size_t offset, size_t length) const;
+  XErrno fallocate_fill_hole_syscall(size_t offset, size_t length) const;
+  XErrno fallocate_fill_hole(size_t offset, size_t length) const;
+  XErrno fallocate_punch_hole(size_t offset, size_t length) const;
+  XErrno split_and_fallocate(bool punch_hole, size_t offset, size_t length) const;
+  XErrno fallocate(bool punch_hole, size_t offset, size_t length) const;
+
+  bool commit_inner(size_t offset, size_t length) const;
+  size_t commit_numa_interleaved(size_t offset, size_t length) const;
+  size_t commit_default(size_t offset, size_t length) const;
+
+public:
+  XPhysicalMemoryBacking(size_t max_capacity);
+
+  bool is_initialized() const;
+
+  void warn_commit_limits(size_t max_capacity) const;
+
+  size_t commit(size_t offset, size_t length) const;
+  size_t uncommit(size_t offset, size_t length) const;
+
+  void map(uintptr_t addr, size_t size, uintptr_t offset) const;
+  void unmap(uintptr_t addr, size_t size) const;
+};
+
+#endif // OS_LINUX_GC_X_XPHYSICALMEMORYBACKING_LINUX_HPP
diff --git a/src/hotspot/os/linux/gc/x/xSyscall_linux.cpp b/src/hotspot/os/linux/gc/x/xSyscall_linux.cpp
new file mode 100644
index 00000000000..6035eaae61b
--- /dev/null
+++ b/src/hotspot/os/linux/gc/x/xSyscall_linux.cpp
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xSyscall_linux.hpp"
+#include OS_CPU_HEADER(gc/x/xSyscall)
+
+#include <unistd.h>
+
+int XSyscall::memfd_create(const char *name, unsigned int flags) {
+  return syscall(SYS_memfd_create, name, flags);
+}
+
+int XSyscall::fallocate(int fd, int mode, size_t offset, size_t length) {
+  return syscall(SYS_fallocate, fd, mode, offset, length);
+}
+
+long XSyscall::get_mempolicy(int* mode, unsigned long* nodemask, unsigned long maxnode, void* addr, unsigned long flags) {
+  return syscall(SYS_get_mempolicy, mode, nodemask, maxnode, addr, flags);
+}
diff --git a/src/hotspot/os/linux/gc/x/xSyscall_linux.hpp b/src/hotspot/os/linux/gc/x/xSyscall_linux.hpp
new file mode 100644
index 00000000000..f16d2b2ffdc
--- /dev/null
+++ b/src/hotspot/os/linux/gc/x/xSyscall_linux.hpp
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef OS_LINUX_GC_X_XSYSCALL_LINUX_HPP
+#define OS_LINUX_GC_X_XSYSCALL_LINUX_HPP
+
+#include "memory/allStatic.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+// Flags for get_mempolicy()
+#ifndef MPOL_F_NODE
+#define MPOL_F_NODE        (1<<0)
+#endif
+#ifndef MPOL_F_ADDR
+#define MPOL_F_ADDR        (1<<1)
+#endif
+
+class XSyscall : public AllStatic {
+public:
+  static int memfd_create(const char* name, unsigned int flags);
+  static int fallocate(int fd, int mode, size_t offset, size_t length);
+  static long get_mempolicy(int* mode, unsigned long* nodemask, unsigned long maxnode, void* addr, unsigned long flags);
+};
+
+#endif // OS_LINUX_GC_X_XSYSCALL_LINUX_HPP
diff --git a/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp b/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp
index f2fff578176..bed82b19148 100644
--- a/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp
+++ b/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.cpp
@@ -23,6 +23,7 @@
 
 #include "precompiled.hpp"
 #include "gc/shared/gcLogPrecious.hpp"
+#include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zArray.inline.hpp"
 #include "gc/z/zErrno.hpp"
 #include "gc/z/zGlobals.hpp"
@@ -385,11 +386,11 @@ bool ZPhysicalMemoryBacking::tmpfs_supports_transparent_huge_pages() const {
   return access(ZFILENAME_SHMEM_ENABLED, R_OK) == 0;
 }
 
-ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_hugetlbfs(size_t offset, size_t length, bool touch) const {
+ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_hugetlbfs(zoffset offset, size_t length, bool touch) const {
   // On hugetlbfs, mapping a file segment will fail immediately, without
   // the need to touch the mapped pages first, if there aren't enough huge
   // pages available to back the mapping.
-  void* const addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset);
+  void* const addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, untype(offset));
   if (addr == MAP_FAILED) {
     // Failed
     return errno;
@@ -436,10 +437,10 @@ static bool safe_touch_mapping(void* addr, size_t length, size_t page_size) {
   return true;
 }
 
-ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_tmpfs(size_t offset, size_t length) const {
+ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_tmpfs(zoffset offset, size_t length) const {
   // On tmpfs, we need to touch the mapped pages to figure out
   // if there are enough pages available to back the mapping.
-  void* const addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, offset);
+  void* const addr = mmap(0, length, PROT_READ|PROT_WRITE, MAP_SHARED, _fd, untype(offset));
   if (addr == MAP_FAILED) {
     // Failed
     return errno;
@@ -463,12 +464,12 @@ ZErrno ZPhysicalMemoryBacking::fallocate_compat_mmap_tmpfs(size_t offset, size_t
   return backed ? 0 : ENOMEM;
 }
 
-ZErrno ZPhysicalMemoryBacking::fallocate_compat_pwrite(size_t offset, size_t length) const {
+ZErrno ZPhysicalMemoryBacking::fallocate_compat_pwrite(zoffset offset, size_t length) const {
   uint8_t data = 0;
 
   // Allocate backing memory by writing to each block
-  for (size_t pos = offset; pos < offset + length; pos += _block_size) {
-    if (pwrite(_fd, &data, sizeof(data), pos) == -1) {
+  for (zoffset pos = offset; pos < offset + length; pos += _block_size) {
+    if (pwrite(_fd, &data, sizeof(data), untype(pos)) == -1) {
       // Failed
       return errno;
     }
@@ -478,7 +479,7 @@ ZErrno ZPhysicalMemoryBacking::fallocate_compat_pwrite(size_t offset, size_t len
   return 0;
 }
 
-ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole_compat(size_t offset, size_t length) const {
+ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole_compat(zoffset offset, size_t length) const {
   // fallocate(2) is only supported by tmpfs since Linux 3.5, and by hugetlbfs
   // since Linux 4.3. When fallocate(2) is not supported we emulate it using
   // mmap/munmap (for hugetlbfs and tmpfs with transparent huge pages) or pwrite
@@ -492,9 +493,9 @@ ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole_compat(size_t offset, size_t
   }
 }
 
-ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole_syscall(size_t offset, size_t length) const {
+ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole_syscall(zoffset offset, size_t length) const {
   const int mode = 0; // Allocate
-  const int res = ZSyscall::fallocate(_fd, mode, offset, length);
+  const int res = ZSyscall::fallocate(_fd, mode, untype(offset), length);
   if (res == -1) {
     // Failed
     return errno;
@@ -504,7 +505,7 @@ ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole_syscall(size_t offset, size_t
   return 0;
 }
 
-ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole(size_t offset, size_t length) const {
+ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole(zoffset offset, size_t length) const {
   // Using compat mode is more efficient when allocating space on hugetlbfs.
   // Note that allocating huge pages this way will only reserve them, and not
   // associate them with segments of the file. We must guarantee that we at
@@ -531,7 +532,7 @@ ZErrno ZPhysicalMemoryBacking::fallocate_fill_hole(size_t offset, size_t length)
   return fallocate_fill_hole_compat(offset, length);
 }
 
-ZErrno ZPhysicalMemoryBacking::fallocate_punch_hole(size_t offset, size_t length) const {
+ZErrno ZPhysicalMemoryBacking::fallocate_punch_hole(zoffset offset, size_t length) const {
   if (ZLargePages::is_explicit()) {
     // We can only punch hole in pages that have been touched. Non-touched
     // pages are only reserved, and not associated with any specific file
@@ -545,7 +546,7 @@ ZErrno ZPhysicalMemoryBacking::fallocate_punch_hole(size_t offset, size_t length
   }
 
   const int mode = FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE;
-  if (ZSyscall::fallocate(_fd, mode, offset, length) == -1) {
+  if (ZSyscall::fallocate(_fd, mode, untype(offset), length) == -1) {
     // Failed
     return errno;
   }
@@ -554,9 +555,9 @@ ZErrno ZPhysicalMemoryBacking::fallocate_punch_hole(size_t offset, size_t length
   return 0;
 }
 
-ZErrno ZPhysicalMemoryBacking::split_and_fallocate(bool punch_hole, size_t offset, size_t length) const {
+ZErrno ZPhysicalMemoryBacking::split_and_fallocate(bool punch_hole, zoffset offset, size_t length) const {
   // Try first half
-  const size_t offset0 = offset;
+  const zoffset offset0 = offset;
   const size_t length0 = align_up(length / 2, _block_size);
   const ZErrno err0 = fallocate(punch_hole, offset0, length0);
   if (err0) {
@@ -564,7 +565,7 @@ ZErrno ZPhysicalMemoryBacking::split_and_fallocate(bool punch_hole, size_t offse
   }
 
   // Try second half
-  const size_t offset1 = offset0 + length0;
+  const zoffset offset1 = offset0 + length0;
   const size_t length1 = length - length0;
   const ZErrno err1 = fallocate(punch_hole, offset1, length1);
   if (err1) {
@@ -575,8 +576,8 @@ ZErrno ZPhysicalMemoryBacking::split_and_fallocate(bool punch_hole, size_t offse
   return 0;
 }
 
-ZErrno ZPhysicalMemoryBacking::fallocate(bool punch_hole, size_t offset, size_t length) const {
-  assert(is_aligned(offset, _block_size), "Invalid offset");
+ZErrno ZPhysicalMemoryBacking::fallocate(bool punch_hole, zoffset offset, size_t length) const {
+  assert(is_aligned(untype(offset), _block_size), "Invalid offset");
   assert(is_aligned(length, _block_size), "Invalid length");
 
   const ZErrno err = punch_hole ? fallocate_punch_hole(offset, length) : fallocate_fill_hole(offset, length);
@@ -591,9 +592,9 @@ ZErrno ZPhysicalMemoryBacking::fallocate(bool punch_hole, size_t offset, size_t
   return err;
 }
 
-bool ZPhysicalMemoryBacking::commit_inner(size_t offset, size_t length) const {
+bool ZPhysicalMemoryBacking::commit_inner(zoffset offset, size_t length) const {
   log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
-                      offset / M, (offset + length) / M, length / M);
+                      untype(offset) / M, untype(offset + length) / M, length / M);
 
 retry:
   const ZErrno err = fallocate(false /* punch_hole */, offset, length);
@@ -622,19 +623,19 @@ retry:
   return true;
 }
 
-static int offset_to_node(size_t offset) {
+static int offset_to_node(zoffset offset) {
   const GrowableArray<int>* mapping = os::Linux::numa_nindex_to_node();
-  const size_t nindex = (offset >> ZGranuleSizeShift) % mapping->length();
+  const size_t nindex = (untype(offset) >> ZGranuleSizeShift) % mapping->length();
   return mapping->at((int)nindex);
 }
 
-size_t ZPhysicalMemoryBacking::commit_numa_interleaved(size_t offset, size_t length) const {
+size_t ZPhysicalMemoryBacking::commit_numa_interleaved(zoffset offset, size_t length) const {
   size_t committed = 0;
 
   // Commit one granule at a time, so that each granule
   // can be allocated from a different preferred node.
   while (committed < length) {
-    const size_t granule_offset = offset + committed;
+    const zoffset granule_offset = offset + committed;
 
     // Setup NUMA policy to allocate memory from a preferred node
     os::Linux::numa_set_preferred(offset_to_node(granule_offset));
@@ -653,7 +654,7 @@ size_t ZPhysicalMemoryBacking::commit_numa_interleaved(size_t offset, size_t len
   return committed;
 }
 
-size_t ZPhysicalMemoryBacking::commit_default(size_t offset, size_t length) const {
+size_t ZPhysicalMemoryBacking::commit_default(zoffset offset, size_t length) const {
   // Try to commit the whole region
   if (commit_inner(offset, length)) {
     // Success
@@ -661,8 +662,8 @@ size_t ZPhysicalMemoryBacking::commit_default(size_t offset, size_t length) cons
   }
 
   // Failed, try to commit as much as possible
-  size_t start = offset;
-  size_t end = offset + length;
+  zoffset start = offset;
+  zoffset end = offset + length;
 
   for (;;) {
     length = align_down((end - start) / 2, ZGranuleSize);
@@ -681,7 +682,7 @@ size_t ZPhysicalMemoryBacking::commit_default(size_t offset, size_t length) cons
   }
 }
 
-size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) const {
+size_t ZPhysicalMemoryBacking::commit(zoffset offset, size_t length) const {
   if (ZNUMA::is_enabled() && !ZLargePages::is_explicit()) {
     // To get granule-level NUMA interleaving when using non-large pages,
     // we must explicitly interleave the memory at commit/fallocate time.
@@ -691,9 +692,9 @@ size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) const {
   return commit_default(offset, length);
 }
 
-size_t ZPhysicalMemoryBacking::uncommit(size_t offset, size_t length) const {
+size_t ZPhysicalMemoryBacking::uncommit(zoffset offset, size_t length) const {
   log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
-                      offset / M, (offset + length) / M, length / M);
+                      untype(offset) / M, untype(offset + length) / M, length / M);
 
   const ZErrno err = fallocate(true /* punch_hole */, offset, length);
   if (err) {
@@ -704,19 +705,19 @@ size_t ZPhysicalMemoryBacking::uncommit(size_t offset, size_t length) const {
   return length;
 }
 
-void ZPhysicalMemoryBacking::map(uintptr_t addr, size_t size, uintptr_t offset) const {
-  const void* const res = mmap((void*)addr, size, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _fd, offset);
+void ZPhysicalMemoryBacking::map(zaddress_unsafe addr, size_t size, zoffset offset) const {
+  const void* const res = mmap((void*)untype(addr), size, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_SHARED, _fd, untype(offset));
   if (res == MAP_FAILED) {
     ZErrno err;
     fatal("Failed to map memory (%s)", err.to_string());
   }
 }
 
-void ZPhysicalMemoryBacking::unmap(uintptr_t addr, size_t size) const {
+void ZPhysicalMemoryBacking::unmap(zaddress_unsafe addr, size_t size) const {
   // Note that we must keep the address space reservation intact and just detach
   // the backing memory. For this reason we map a new anonymous, non-accessible
   // and non-reserved page over the mapping instead of actually unmapping.
-  const void* const res = mmap((void*)addr, size, PROT_NONE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
+  const void* const res = mmap((void*)untype(addr), size, PROT_NONE, MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, -1, 0);
   if (res == MAP_FAILED) {
     ZErrno err;
     fatal("Failed to map memory (%s)", err.to_string());
diff --git a/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.hpp b/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.hpp
index 655c5a4209b..59c00ad01bf 100644
--- a/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.hpp
+++ b/src/hotspot/os/linux/gc/z/zPhysicalMemoryBacking_linux.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,8 @@
 #ifndef OS_LINUX_GC_Z_ZPHYSICALMEMORYBACKING_LINUX_HPP
 #define OS_LINUX_GC_Z_ZPHYSICALMEMORYBACKING_LINUX_HPP
 
+#include "gc/z/zAddress.hpp"
+
 class ZErrno;
 
 class ZPhysicalMemoryBacking {
@@ -46,19 +48,19 @@ private:
   bool is_hugetlbfs() const;
   bool tmpfs_supports_transparent_huge_pages() const;
 
-  ZErrno fallocate_compat_mmap_hugetlbfs(size_t offset, size_t length, bool touch) const;
-  ZErrno fallocate_compat_mmap_tmpfs(size_t offset, size_t length) const;
-  ZErrno fallocate_compat_pwrite(size_t offset, size_t length) const;
-  ZErrno fallocate_fill_hole_compat(size_t offset, size_t length) const;
-  ZErrno fallocate_fill_hole_syscall(size_t offset, size_t length) const;
-  ZErrno fallocate_fill_hole(size_t offset, size_t length) const;
-  ZErrno fallocate_punch_hole(size_t offset, size_t length) const;
-  ZErrno split_and_fallocate(bool punch_hole, size_t offset, size_t length) const;
-  ZErrno fallocate(bool punch_hole, size_t offset, size_t length) const;
+  ZErrno fallocate_compat_mmap_hugetlbfs(zoffset offset, size_t length, bool touch) const;
+  ZErrno fallocate_compat_mmap_tmpfs(zoffset offset, size_t length) const;
+  ZErrno fallocate_compat_pwrite(zoffset offset, size_t length) const;
+  ZErrno fallocate_fill_hole_compat(zoffset offset, size_t length) const;
+  ZErrno fallocate_fill_hole_syscall(zoffset offset, size_t length) const;
+  ZErrno fallocate_fill_hole(zoffset offset, size_t length) const;
+  ZErrno fallocate_punch_hole(zoffset offset, size_t length) const;
+  ZErrno split_and_fallocate(bool punch_hole, zoffset offset, size_t length) const;
+  ZErrno fallocate(bool punch_hole, zoffset offset, size_t length) const;
 
-  bool commit_inner(size_t offset, size_t length) const;
-  size_t commit_numa_interleaved(size_t offset, size_t length) const;
-  size_t commit_default(size_t offset, size_t length) const;
+  bool commit_inner(zoffset offset, size_t length) const;
+  size_t commit_numa_interleaved(zoffset offset, size_t length) const;
+  size_t commit_default(zoffset offset, size_t length) const;
 
 public:
   ZPhysicalMemoryBacking(size_t max_capacity);
@@ -67,11 +69,11 @@ public:
 
   void warn_commit_limits(size_t max_capacity) const;
 
-  size_t commit(size_t offset, size_t length) const;
-  size_t uncommit(size_t offset, size_t length) const;
+  size_t commit(zoffset offset, size_t length) const;
+  size_t uncommit(zoffset offset, size_t length) const;
 
-  void map(uintptr_t addr, size_t size, uintptr_t offset) const;
-  void unmap(uintptr_t addr, size_t size) const;
+  void map(zaddress_unsafe addr, size_t size, zoffset offset) const;
+  void unmap(zaddress_unsafe addr, size_t size) const;
 };
 
 #endif // OS_LINUX_GC_Z_ZPHYSICALMEMORYBACKING_LINUX_HPP
diff --git a/src/hotspot/os/posix/gc/x/xArguments_posix.cpp b/src/hotspot/os/posix/gc/x/xArguments_posix.cpp
new file mode 100644
index 00000000000..6df0a9bd074
--- /dev/null
+++ b/src/hotspot/os/posix/gc/x/xArguments_posix.cpp
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xArguments.hpp"
+
+bool XArguments::is_os_supported() {
+  return true;
+}
diff --git a/src/hotspot/os/posix/gc/x/xInitialize_posix.cpp b/src/hotspot/os/posix/gc/x/xInitialize_posix.cpp
new file mode 100644
index 00000000000..acf71e98901
--- /dev/null
+++ b/src/hotspot/os/posix/gc/x/xInitialize_posix.cpp
@@ -0,0 +1,29 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xInitialize.hpp"
+
+void XInitialize::pd_initialize() {
+  // Does nothing
+}
diff --git a/src/hotspot/os/posix/gc/x/xUtils_posix.cpp b/src/hotspot/os/posix/gc/x/xUtils_posix.cpp
new file mode 100644
index 00000000000..eee3e5cfbe6
--- /dev/null
+++ b/src/hotspot/os/posix/gc/x/xUtils_posix.cpp
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xUtils.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+#include <stdlib.h>
+
+uintptr_t XUtils::alloc_aligned(size_t alignment, size_t size) {
+  void* res = nullptr;
+
+  // Use raw posix_memalign as long as we have no wrapper for it
+  ALLOW_C_FUNCTION(::posix_memalign, int rc = posix_memalign(&res, alignment, size);)
+  if (rc != 0) {
+    fatal("posix_memalign() failed");
+  }
+
+  memset(res, 0, size);
+
+  return (uintptr_t)res;
+}
diff --git a/src/hotspot/os/posix/gc/x/xVirtualMemory_posix.cpp b/src/hotspot/os/posix/gc/x/xVirtualMemory_posix.cpp
new file mode 100644
index 00000000000..e2422eb0978
--- /dev/null
+++ b/src/hotspot/os/posix/gc/x/xVirtualMemory_posix.cpp
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xVirtualMemory.hpp"
+#include "logging/log.hpp"
+
+#include <sys/mman.h>
+#include <sys/types.h>
+
+void XVirtualMemoryManager::pd_initialize_before_reserve() {
+  // Does nothing
+}
+
+void XVirtualMemoryManager::pd_initialize_after_reserve() {
+  // Does nothing
+}
+
+bool XVirtualMemoryManager::pd_reserve(uintptr_t addr, size_t size) {
+  const uintptr_t res = (uintptr_t)mmap((void*)addr, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0);
+  if (res == (uintptr_t)MAP_FAILED) {
+    // Failed to reserve memory
+    return false;
+  }
+
+  if (res != addr) {
+    // Failed to reserve memory at the requested address
+    munmap((void*)res, size);
+    return false;
+  }
+
+  // Success
+  return true;
+}
+
+void XVirtualMemoryManager::pd_unreserve(uintptr_t addr, size_t size) {
+  const int res = munmap((void*)addr, size);
+  assert(res == 0, "Failed to unmap memory");
+}
diff --git a/src/hotspot/os/posix/gc/z/zArguments_posix.cpp b/src/hotspot/os/posix/gc/z/zArguments_posix.cpp
index fd91c6a7d7b..4e6d43b16e9 100644
--- a/src/hotspot/os/posix/gc/z/zArguments_posix.cpp
+++ b/src/hotspot/os/posix/gc/z/zArguments_posix.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,6 @@
 #include "precompiled.hpp"
 #include "gc/z/zArguments.hpp"
 
-bool ZArguments::is_os_supported() const {
+bool ZArguments::is_os_supported() {
   return true;
 }
diff --git a/src/hotspot/os/posix/gc/z/zVirtualMemory_posix.cpp b/src/hotspot/os/posix/gc/z/zVirtualMemory_posix.cpp
index 19dfed891da..936e734e8ff 100644
--- a/src/hotspot/os/posix/gc/z/zVirtualMemory_posix.cpp
+++ b/src/hotspot/os/posix/gc/z/zVirtualMemory_posix.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -37,16 +37,16 @@ void ZVirtualMemoryManager::pd_initialize_after_reserve() {
   // Does nothing
 }
 
-bool ZVirtualMemoryManager::pd_reserve(uintptr_t addr, size_t size) {
-  const uintptr_t res = (uintptr_t)mmap((void*)addr, size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0);
-  if (res == (uintptr_t)MAP_FAILED) {
+bool ZVirtualMemoryManager::pd_reserve(zaddress_unsafe addr, size_t size) {
+  void* const res = mmap((void*)untype(addr), size, PROT_NONE, MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0);
+  if (res == MAP_FAILED) {
     // Failed to reserve memory
     return false;
   }
 
-  if (res != addr) {
+  if (res != (void*)untype(addr)) {
     // Failed to reserve memory at the requested address
-    munmap((void*)res, size);
+    munmap(res, size);
     return false;
   }
 
@@ -54,7 +54,7 @@ bool ZVirtualMemoryManager::pd_reserve(uintptr_t addr, size_t size) {
   return true;
 }
 
-void ZVirtualMemoryManager::pd_unreserve(uintptr_t addr, size_t size) {
-  const int res = munmap((void*)addr, size);
+void ZVirtualMemoryManager::pd_unreserve(zaddress_unsafe addr, size_t size) {
+  const int res = munmap((void*)untype(addr), size);
   assert(res == 0, "Failed to unmap memory");
 }
diff --git a/src/hotspot/os/windows/gc/x/xArguments_windows.cpp b/src/hotspot/os/windows/gc/x/xArguments_windows.cpp
new file mode 100644
index 00000000000..fc5f7eccb91
--- /dev/null
+++ b/src/hotspot/os/windows/gc/x/xArguments_windows.cpp
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xArguments.hpp"
+#include "gc/x/xSyscall_windows.hpp"
+
+bool XArguments::is_os_supported() {
+  return XSyscall::is_supported();
+}
diff --git a/src/hotspot/os/windows/gc/x/xInitialize_windows.cpp b/src/hotspot/os/windows/gc/x/xInitialize_windows.cpp
new file mode 100644
index 00000000000..99f64328033
--- /dev/null
+++ b/src/hotspot/os/windows/gc/x/xInitialize_windows.cpp
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xInitialize.hpp"
+#include "gc/x/xSyscall_windows.hpp"
+
+void XInitialize::pd_initialize() {
+  XSyscall::initialize();
+}
diff --git a/src/hotspot/os/windows/gc/x/xLargePages_windows.cpp b/src/hotspot/os/windows/gc/x/xLargePages_windows.cpp
new file mode 100644
index 00000000000..20b3c4911fc
--- /dev/null
+++ b/src/hotspot/os/windows/gc/x/xLargePages_windows.cpp
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/x/xLargePages.hpp"
+#include "gc/x/xSyscall_windows.hpp"
+#include "runtime/globals.hpp"
+
+void XLargePages::pd_initialize() {
+  if (UseLargePages) {
+    if (XSyscall::is_large_pages_supported()) {
+      _state = Explicit;
+      return;
+    }
+    log_info_p(gc, init)("Shared large pages not supported on this OS version");
+  }
+
+  _state = Disabled;
+}
diff --git a/src/hotspot/os/windows/gc/x/xMapper_windows.cpp b/src/hotspot/os/windows/gc/x/xMapper_windows.cpp
new file mode 100644
index 00000000000..e69b6ec56e2
--- /dev/null
+++ b/src/hotspot/os/windows/gc/x/xMapper_windows.cpp
@@ -0,0 +1,310 @@
+/*
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xMapper_windows.hpp"
+#include "gc/x/xSyscall_windows.hpp"
+#include "logging/log.hpp"
+#include "utilities/debug.hpp"
+
+#include <Windows.h>
+
+// Memory reservation, commit, views, and placeholders.
+//
+// To be able to up-front reserve address space for the heap views, and later
+// multi-map the heap views to the same physical memory, without ever losing the
+// reservation of the reserved address space, we use "placeholders".
+//
+// These placeholders block out the address space from being used by other parts
+// of the process. To commit memory in this address space, the placeholder must
+// be replaced by anonymous memory, or replaced by mapping a view against a
+// paging file mapping. We use the later to support multi-mapping.
+//
+// We want to be able to dynamically commit and uncommit the physical memory of
+// the heap (and also unmap ZPages), in granules of ZGranuleSize bytes. There is
+// no way to grow and shrink the committed memory of a paging file mapping.
+// Therefore, we create multiple granule-sized page file mappings. The memory is
+// committed by creating a page file mapping, map a view against it, commit the
+// memory, unmap the view. The memory will stay committed until all views are
+// unmapped, and the paging file mapping handle is closed.
+//
+// When replacing a placeholder address space reservation with a mapped view
+// against a paging file mapping, the virtual address space must exactly match
+// an existing placeholder's address and size. Therefore we only deal with
+// granule-sized placeholders at this layer. Higher layers that keep track of
+// reserved available address space can (and will) coalesce placeholders, but
+// they will be split before being used.
+
+#define fatal_error(msg, addr, size)                  \
+  fatal(msg ": " PTR_FORMAT " " SIZE_FORMAT "M (%d)", \
+        (addr), (size) / M, GetLastError())
+
+uintptr_t XMapper::reserve(uintptr_t addr, size_t size) {
+  void* const res = XSyscall::VirtualAlloc2(
+    GetCurrentProcess(),                   // Process
+    (void*)addr,                           // BaseAddress
+    size,                                  // Size
+    MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, // AllocationType
+    PAGE_NOACCESS,                         // PageProtection
+    nullptr,                               // ExtendedParameters
+    0                                      // ParameterCount
+    );
+
+  // Caller responsible for error handling
+  return (uintptr_t)res;
+}
+
+void XMapper::unreserve(uintptr_t addr, size_t size) {
+  const bool res = XSyscall::VirtualFreeEx(
+    GetCurrentProcess(), // hProcess
+    (void*)addr,         // lpAddress
+    size,                // dwSize
+    MEM_RELEASE          // dwFreeType
+    );
+
+  if (!res) {
+    fatal_error("Failed to unreserve memory", addr, size);
+  }
+}
+
+HANDLE XMapper::create_paging_file_mapping(size_t size) {
+  // Create mapping with SEC_RESERVE instead of SEC_COMMIT.
+  //
+  // We use MapViewOfFile3 for two different reasons:
+  //  1) When committing memory for the created paging file
+  //  2) When mapping a view of the memory created in (2)
+  //
+  // The non-platform code is only setup to deal with out-of-memory
+  // errors in (1). By using SEC_RESERVE, we prevent MapViewOfFile3
+  // from failing because of "commit limit" checks. To actually commit
+  // memory in (1), a call to VirtualAlloc2 is done.
+
+  HANDLE const res = XSyscall::CreateFileMappingW(
+    INVALID_HANDLE_VALUE,         // hFile
+    nullptr,                      // lpFileMappingAttribute
+    PAGE_READWRITE | SEC_RESERVE, // flProtect
+    size >> 32,                   // dwMaximumSizeHigh
+    size & 0xFFFFFFFF,            // dwMaximumSizeLow
+    nullptr                       // lpName
+    );
+
+  // Caller responsible for error handling
+  return res;
+}
+
+bool XMapper::commit_paging_file_mapping(HANDLE file_handle, uintptr_t file_offset, size_t size) {
+  const uintptr_t addr = map_view_no_placeholder(file_handle, file_offset, size);
+  if (addr == 0) {
+    log_error(gc)("Failed to map view of paging file mapping (%d)", GetLastError());
+    return false;
+  }
+
+  const uintptr_t res = commit(addr, size);
+  if (res != addr) {
+    log_error(gc)("Failed to commit memory (%d)", GetLastError());
+  }
+
+  unmap_view_no_placeholder(addr, size);
+
+  return res == addr;
+}
+
+uintptr_t XMapper::map_view_no_placeholder(HANDLE file_handle, uintptr_t file_offset, size_t size) {
+  void* const res = XSyscall::MapViewOfFile3(
+    file_handle,         // FileMapping
+    GetCurrentProcess(), // ProcessHandle
+    nullptr,             // BaseAddress
+    file_offset,         // Offset
+    size,                // ViewSize
+    0,                   // AllocationType
+    PAGE_NOACCESS,       // PageProtection
+    nullptr,             // ExtendedParameters
+    0                    // ParameterCount
+    );
+
+  // Caller responsible for error handling
+  return (uintptr_t)res;
+}
+
+void XMapper::unmap_view_no_placeholder(uintptr_t addr, size_t size) {
+  const bool res = XSyscall::UnmapViewOfFile2(
+    GetCurrentProcess(), // ProcessHandle
+    (void*)addr,         // BaseAddress
+    0                    // UnmapFlags
+    );
+
+  if (!res) {
+    fatal_error("Failed to unmap memory", addr, size);
+  }
+}
+
+uintptr_t XMapper::commit(uintptr_t addr, size_t size) {
+  void* const res = XSyscall::VirtualAlloc2(
+    GetCurrentProcess(), // Process
+    (void*)addr,         // BaseAddress
+    size,                // Size
+    MEM_COMMIT,          // AllocationType
+    PAGE_NOACCESS,       // PageProtection
+    nullptr,             // ExtendedParameters
+    0                    // ParameterCount
+    );
+
+  // Caller responsible for error handling
+  return (uintptr_t)res;
+}
+
+HANDLE XMapper::create_and_commit_paging_file_mapping(size_t size) {
+  HANDLE const file_handle = create_paging_file_mapping(size);
+  if (file_handle == 0) {
+    log_error(gc)("Failed to create paging file mapping (%d)", GetLastError());
+    return 0;
+  }
+
+  const bool res = commit_paging_file_mapping(file_handle, 0 /* file_offset */, size);
+  if (!res) {
+    close_paging_file_mapping(file_handle);
+    return 0;
+  }
+
+  return file_handle;
+}
+
+void XMapper::close_paging_file_mapping(HANDLE file_handle) {
+  const bool res = CloseHandle(
+    file_handle // hObject
+    );
+
+  if (!res) {
+    fatal("Failed to close paging file handle (%d)", GetLastError());
+  }
+}
+
+HANDLE XMapper::create_shared_awe_section() {
+  MEM_EXTENDED_PARAMETER parameter = { 0 };
+  parameter.Type = MemSectionExtendedParameterUserPhysicalFlags;
+  parameter.ULong64 = 0;
+
+  HANDLE section = XSyscall::CreateFileMapping2(
+    INVALID_HANDLE_VALUE,                 // File
+    nullptr,                              // SecurityAttributes
+    SECTION_MAP_READ | SECTION_MAP_WRITE, // DesiredAccess
+    PAGE_READWRITE,                       // PageProtection
+    SEC_RESERVE | SEC_LARGE_PAGES,        // AllocationAttributes
+    0,                                    // MaximumSize
+    nullptr,                              // Name
+    &parameter,                           // ExtendedParameters
+    1                                     // ParameterCount
+    );
+
+  if (section == nullptr) {
+    fatal("Could not create shared AWE section (%d)", GetLastError());
+  }
+
+  return section;
+}
+
+uintptr_t XMapper::reserve_for_shared_awe(HANDLE awe_section, uintptr_t addr, size_t size) {
+  MEM_EXTENDED_PARAMETER parameter = { 0 };
+  parameter.Type = MemExtendedParameterUserPhysicalHandle;
+  parameter.Handle = awe_section;
+
+  void* const res = XSyscall::VirtualAlloc2(
+    GetCurrentProcess(),        // Process
+    (void*)addr,                // BaseAddress
+    size,                       // Size
+    MEM_RESERVE | MEM_PHYSICAL, // AllocationType
+    PAGE_READWRITE,             // PageProtection
+    &parameter,                 // ExtendedParameters
+    1                           // ParameterCount
+    );
+
+  // Caller responsible for error handling
+  return (uintptr_t)res;
+}
+
+void XMapper::unreserve_for_shared_awe(uintptr_t addr, size_t size) {
+  bool res = VirtualFree(
+    (void*)addr, // lpAddress
+    0,           // dwSize
+    MEM_RELEASE  // dwFreeType
+    );
+
+  if (!res) {
+    fatal("Failed to unreserve memory: " PTR_FORMAT " " SIZE_FORMAT "M (%d)",
+          addr, size / M, GetLastError());
+  }
+}
+
+void XMapper::split_placeholder(uintptr_t addr, size_t size) {
+  const bool res = VirtualFree(
+    (void*)addr,                           // lpAddress
+    size,                                  // dwSize
+    MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER // dwFreeType
+    );
+
+  if (!res) {
+    fatal_error("Failed to split placeholder", addr, size);
+  }
+}
+
+void XMapper::coalesce_placeholders(uintptr_t addr, size_t size) {
+  const bool res = VirtualFree(
+    (void*)addr,                            // lpAddress
+    size,                                   // dwSize
+    MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS // dwFreeType
+    );
+
+  if (!res) {
+    fatal_error("Failed to coalesce placeholders", addr, size);
+  }
+}
+
+void XMapper::map_view_replace_placeholder(HANDLE file_handle, uintptr_t file_offset, uintptr_t addr, size_t size) {
+  void* const res = XSyscall::MapViewOfFile3(
+    file_handle,             // FileMapping
+    GetCurrentProcess(),     // ProcessHandle
+    (void*)addr,             // BaseAddress
+    file_offset,             // Offset
+    size,                    // ViewSize
+    MEM_REPLACE_PLACEHOLDER, // AllocationType
+    PAGE_READWRITE,          // PageProtection
+    nullptr,                 // ExtendedParameters
+    0                        // ParameterCount
+    );
+
+  if (res == nullptr) {
+    fatal_error("Failed to map memory", addr, size);
+  }
+}
+
+void XMapper::unmap_view_preserve_placeholder(uintptr_t addr, size_t size) {
+  const bool res = XSyscall::UnmapViewOfFile2(
+    GetCurrentProcess(),     // ProcessHandle
+    (void*)addr,             // BaseAddress
+    MEM_PRESERVE_PLACEHOLDER // UnmapFlags
+    );
+
+  if (!res) {
+    fatal_error("Failed to unmap memory", addr, size);
+  }
+}
diff --git a/src/hotspot/os/windows/gc/x/xMapper_windows.hpp b/src/hotspot/os/windows/gc/x/xMapper_windows.hpp
new file mode 100644
index 00000000000..0f266d3fab7
--- /dev/null
+++ b/src/hotspot/os/windows/gc/x/xMapper_windows.hpp
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef OS_WINDOWS_GC_X_XMAPPER_WINDOWS_HPP
+#define OS_WINDOWS_GC_X_XMAPPER_WINDOWS_HPP
+
+#include "memory/allStatic.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+#include <Windows.h>
+
+class XMapper : public AllStatic {
+private:
+  // Create paging file mapping
+  static HANDLE create_paging_file_mapping(size_t size);
+
+  // Commit paging file mapping
+  static bool commit_paging_file_mapping(HANDLE file_handle, uintptr_t file_offset, size_t size);
+
+  // Map a view anywhere without a placeholder
+  static uintptr_t map_view_no_placeholder(HANDLE file_handle, uintptr_t file_offset, size_t size);
+
+  // Unmap a view without preserving a placeholder
+  static void unmap_view_no_placeholder(uintptr_t addr, size_t size);
+
+  // Commit memory covering the given virtual address range
+  static uintptr_t commit(uintptr_t addr, size_t size);
+
+public:
+  // Reserve memory with a placeholder
+  static uintptr_t reserve(uintptr_t addr, size_t size);
+
+  // Unreserve memory
+  static void unreserve(uintptr_t addr, size_t size);
+
+  // Create and commit paging file mapping
+  static HANDLE create_and_commit_paging_file_mapping(size_t size);
+
+  // Close paging file mapping
+  static void close_paging_file_mapping(HANDLE file_handle);
+
+  // Create a shared AWE section
+  static HANDLE create_shared_awe_section();
+
+  // Reserve memory attached to the shared AWE section
+  static uintptr_t reserve_for_shared_awe(HANDLE awe_section, uintptr_t addr, size_t size);
+
+  // Unreserve memory attached to a shared AWE section
+  static void unreserve_for_shared_awe(uintptr_t addr, size_t size);
+
+  // Split a placeholder
+  //
+  // A view can only replace an entire placeholder, so placeholders need to be
+  // split and coalesced to be the exact size of the new views.
+  // [addr, addr + size) needs to be a proper sub-placeholder of an existing
+  // placeholder.
+  static void split_placeholder(uintptr_t addr, size_t size);
+
+  // Coalesce a placeholder
+  //
+  // [addr, addr + size) is the new placeholder. A sub-placeholder needs to
+  // exist within that range.
+  static void coalesce_placeholders(uintptr_t addr, size_t size);
+
+  // Map a view of the file handle and replace the placeholder covering the
+  // given virtual address range
+  static void map_view_replace_placeholder(HANDLE file_handle, uintptr_t file_offset, uintptr_t addr, size_t size);
+
+  // Unmap the view and reinstate a placeholder covering the given virtual
+  // address range
+  static void unmap_view_preserve_placeholder(uintptr_t addr, size_t size);
+};
+
+#endif // OS_WINDOWS_GC_X_XMAPPER_WINDOWS_HPP
diff --git a/src/hotspot/os/windows/gc/x/xNUMA_windows.cpp b/src/hotspot/os/windows/gc/x/xNUMA_windows.cpp
new file mode 100644
index 00000000000..47a84df962e
--- /dev/null
+++ b/src/hotspot/os/windows/gc/x/xNUMA_windows.cpp
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xNUMA.hpp"
+
+void XNUMA::pd_initialize() {
+  _enabled = false;
+}
+
+uint32_t XNUMA::count() {
+  return 1;
+}
+
+uint32_t XNUMA::id() {
+  return 0;
+}
+
+uint32_t XNUMA::memory_id(uintptr_t addr) {
+  // NUMA support not enabled, assume everything belongs to node zero
+  return 0;
+}
diff --git a/src/hotspot/os/windows/gc/x/xPhysicalMemoryBacking_windows.cpp b/src/hotspot/os/windows/gc/x/xPhysicalMemoryBacking_windows.cpp
new file mode 100644
index 00000000000..92d47dfb7c8
--- /dev/null
+++ b/src/hotspot/os/windows/gc/x/xPhysicalMemoryBacking_windows.cpp
@@ -0,0 +1,252 @@
+/*
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xGranuleMap.inline.hpp"
+#include "gc/x/xLargePages.inline.hpp"
+#include "gc/x/xMapper_windows.hpp"
+#include "gc/x/xPhysicalMemoryBacking_windows.hpp"
+#include "logging/log.hpp"
+#include "runtime/globals.hpp"
+#include "utilities/debug.hpp"
+
+class XPhysicalMemoryBackingImpl : public CHeapObj<mtGC> {
+public:
+  virtual size_t commit(size_t offset, size_t size) = 0;
+  virtual size_t uncommit(size_t offset, size_t size) = 0;
+  virtual void map(uintptr_t addr, size_t size, size_t offset) const = 0;
+  virtual void unmap(uintptr_t addr, size_t size) const = 0;
+};
+
+// Implements small pages (paged) support using placeholder reservation.
+//
+// The backing commits and uncommits physical memory, that can be
+// multi-mapped into the virtual address space. To support fine-graned
+// committing and uncommitting, each XGranuleSize'd chunk is mapped to
+// a separate paging file mapping.
+
+class XPhysicalMemoryBackingSmallPages : public XPhysicalMemoryBackingImpl {
+private:
+  XGranuleMap<HANDLE> _handles;
+
+  HANDLE get_handle(uintptr_t offset) const {
+    HANDLE const handle = _handles.get(offset);
+    assert(handle != 0, "Should be set");
+    return handle;
+  }
+
+  void put_handle(uintptr_t offset, HANDLE handle) {
+    assert(handle != INVALID_HANDLE_VALUE, "Invalid handle");
+    assert(_handles.get(offset) == 0, "Should be cleared");
+    _handles.put(offset, handle);
+  }
+
+  void clear_handle(uintptr_t offset) {
+    assert(_handles.get(offset) != 0, "Should be set");
+    _handles.put(offset, 0);
+  }
+
+public:
+  XPhysicalMemoryBackingSmallPages(size_t max_capacity) :
+      XPhysicalMemoryBackingImpl(),
+      _handles(max_capacity) {}
+
+  size_t commit(size_t offset, size_t size) {
+    for (size_t i = 0; i < size; i += XGranuleSize) {
+      HANDLE const handle = XMapper::create_and_commit_paging_file_mapping(XGranuleSize);
+      if (handle == 0) {
+        return i;
+      }
+
+      put_handle(offset + i, handle);
+    }
+
+    return size;
+  }
+
+  size_t uncommit(size_t offset, size_t size) {
+    for (size_t i = 0; i < size; i += XGranuleSize) {
+      HANDLE const handle = get_handle(offset + i);
+      clear_handle(offset + i);
+      XMapper::close_paging_file_mapping(handle);
+    }
+
+    return size;
+  }
+
+  void map(uintptr_t addr, size_t size, size_t offset) const {
+    assert(is_aligned(offset, XGranuleSize), "Misaligned");
+    assert(is_aligned(addr, XGranuleSize), "Misaligned");
+    assert(is_aligned(size, XGranuleSize), "Misaligned");
+
+    for (size_t i = 0; i < size; i += XGranuleSize) {
+      HANDLE const handle = get_handle(offset + i);
+      XMapper::map_view_replace_placeholder(handle, 0 /* offset */, addr + i, XGranuleSize);
+    }
+  }
+
+  void unmap(uintptr_t addr, size_t size) const {
+    assert(is_aligned(addr, XGranuleSize), "Misaligned");
+    assert(is_aligned(size, XGranuleSize), "Misaligned");
+
+    for (size_t i = 0; i < size; i += XGranuleSize) {
+      XMapper::unmap_view_preserve_placeholder(addr + i, XGranuleSize);
+    }
+  }
+};
+
+// Implements Large Pages (locked) support using shared AWE physical memory.
+//
+// Shared AWE physical memory also works with small pages, but it has
+// a few drawbacks that makes it a no-go to use it at this point:
+//
+// 1) It seems to use 8 bytes of committed memory per *reserved* memory.
+// Given our scheme to use a large address space range this turns out to
+// use too much memory.
+//
+// 2) It requires memory locking privileges, even for small pages. This
+// has always been a requirement for large pages, and would be an extra
+// restriction for usage with small pages.
+//
+// Note: The large pages size is tied to our XGranuleSize.
+
+extern HANDLE XAWESection;
+
+class XPhysicalMemoryBackingLargePages : public XPhysicalMemoryBackingImpl {
+private:
+  ULONG_PTR* const _page_array;
+
+  static ULONG_PTR* alloc_page_array(size_t max_capacity) {
+    const size_t npages = max_capacity / XGranuleSize;
+    const size_t array_size = npages * sizeof(ULONG_PTR);
+
+    return (ULONG_PTR*)os::malloc(array_size, mtGC);
+  }
+
+public:
+  XPhysicalMemoryBackingLargePages(size_t max_capacity) :
+      XPhysicalMemoryBackingImpl(),
+      _page_array(alloc_page_array(max_capacity)) {}
+
+  size_t commit(size_t offset, size_t size) {
+    const size_t index = offset >> XGranuleSizeShift;
+    const size_t npages = size >> XGranuleSizeShift;
+
+    size_t npages_res = npages;
+    const bool res = AllocateUserPhysicalPages(XAWESection, &npages_res, &_page_array[index]);
+    if (!res) {
+      fatal("Failed to allocate physical memory " SIZE_FORMAT "M @ " PTR_FORMAT " (%d)",
+            size / M, offset, GetLastError());
+    } else {
+      log_debug(gc)("Allocated physical memory: " SIZE_FORMAT "M @ " PTR_FORMAT, size / M, offset);
+    }
+
+    // AllocateUserPhysicalPages might not be able to allocate the requested amount of memory.
+    // The allocated number of pages are written in npages_res.
+    return npages_res << XGranuleSizeShift;
+  }
+
+  size_t uncommit(size_t offset, size_t size) {
+    const size_t index = offset >> XGranuleSizeShift;
+    const size_t npages = size >> XGranuleSizeShift;
+
+    size_t npages_res = npages;
+    const bool res = FreeUserPhysicalPages(XAWESection, &npages_res, &_page_array[index]);
+    if (!res) {
+      fatal("Failed to uncommit physical memory " SIZE_FORMAT "M @ " PTR_FORMAT " (%d)",
+            size, offset, GetLastError());
+    }
+
+    return npages_res << XGranuleSizeShift;
+  }
+
+  void map(uintptr_t addr, size_t size, size_t offset) const {
+    const size_t npages = size >> XGranuleSizeShift;
+    const size_t index = offset >> XGranuleSizeShift;
+
+    const bool res = MapUserPhysicalPages((char*)addr, npages, &_page_array[index]);
+    if (!res) {
+      fatal("Failed to map view " PTR_FORMAT " " SIZE_FORMAT "M @ " PTR_FORMAT " (%d)",
+            addr, size / M, offset, GetLastError());
+    }
+  }
+
+  void unmap(uintptr_t addr, size_t size) const {
+    const size_t npages = size >> XGranuleSizeShift;
+
+    const bool res = MapUserPhysicalPages((char*)addr, npages, nullptr);
+    if (!res) {
+      fatal("Failed to unmap view " PTR_FORMAT " " SIZE_FORMAT "M (%d)",
+            addr, size / M, GetLastError());
+    }
+  }
+};
+
+static XPhysicalMemoryBackingImpl* select_impl(size_t max_capacity) {
+  if (XLargePages::is_enabled()) {
+    return new XPhysicalMemoryBackingLargePages(max_capacity);
+  }
+
+  return new XPhysicalMemoryBackingSmallPages(max_capacity);
+}
+
+XPhysicalMemoryBacking::XPhysicalMemoryBacking(size_t max_capacity) :
+    _impl(select_impl(max_capacity)) {}
+
+bool XPhysicalMemoryBacking::is_initialized() const {
+  return true;
+}
+
+void XPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const {
+  // Does nothing
+}
+
+size_t XPhysicalMemoryBacking::commit(size_t offset, size_t length) {
+  log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
+                      offset / M, (offset + length) / M, length / M);
+
+  return _impl->commit(offset, length);
+}
+
+size_t XPhysicalMemoryBacking::uncommit(size_t offset, size_t length) {
+  log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
+                      offset / M, (offset + length) / M, length / M);
+
+  return _impl->uncommit(offset, length);
+}
+
+void XPhysicalMemoryBacking::map(uintptr_t addr, size_t size, size_t offset) const {
+  assert(is_aligned(offset, XGranuleSize), "Misaligned: " PTR_FORMAT, offset);
+  assert(is_aligned(addr, XGranuleSize), "Misaligned: " PTR_FORMAT, addr);
+  assert(is_aligned(size, XGranuleSize), "Misaligned: " PTR_FORMAT, size);
+
+  _impl->map(addr, size, offset);
+}
+
+void XPhysicalMemoryBacking::unmap(uintptr_t addr, size_t size) const {
+  assert(is_aligned(addr, XGranuleSize), "Misaligned");
+  assert(is_aligned(size, XGranuleSize), "Misaligned");
+
+  _impl->unmap(addr, size);
+}
diff --git a/src/hotspot/os/windows/gc/x/xPhysicalMemoryBacking_windows.hpp b/src/hotspot/os/windows/gc/x/xPhysicalMemoryBacking_windows.hpp
new file mode 100644
index 00000000000..d6e123f21e5
--- /dev/null
+++ b/src/hotspot/os/windows/gc/x/xPhysicalMemoryBacking_windows.hpp
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef OS_WINDOWS_GC_X_XPHYSICALMEMORYBACKING_WINDOWS_HPP
+#define OS_WINDOWS_GC_X_XPHYSICALMEMORYBACKING_WINDOWS_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+#include <Windows.h>
+
+class XPhysicalMemoryBackingImpl;
+
+class XPhysicalMemoryBacking {
+private:
+  XPhysicalMemoryBackingImpl* _impl;
+
+public:
+  XPhysicalMemoryBacking(size_t max_capacity);
+
+  bool is_initialized() const;
+
+  void warn_commit_limits(size_t max_capacity) const;
+
+  size_t commit(size_t offset, size_t length);
+  size_t uncommit(size_t offset, size_t length);
+
+  void map(uintptr_t addr, size_t size, size_t offset) const;
+  void unmap(uintptr_t addr, size_t size) const;
+};
+
+#endif // OS_WINDOWS_GC_X_XPHYSICALMEMORYBACKING_WINDOWS_HPP
diff --git a/src/hotspot/os/windows/gc/x/xSyscall_windows.cpp b/src/hotspot/os/windows/gc/x/xSyscall_windows.cpp
new file mode 100644
index 00000000000..f22966a5489
--- /dev/null
+++ b/src/hotspot/os/windows/gc/x/xSyscall_windows.cpp
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/x/xSyscall_windows.hpp"
+#include "runtime/java.hpp"
+#include "runtime/os.hpp"
+
+XSyscall::CreateFileMappingWFn XSyscall::CreateFileMappingW;
+XSyscall::CreateFileMapping2Fn XSyscall::CreateFileMapping2;
+XSyscall::VirtualAlloc2Fn XSyscall::VirtualAlloc2;
+XSyscall::VirtualFreeExFn XSyscall::VirtualFreeEx;
+XSyscall::MapViewOfFile3Fn XSyscall::MapViewOfFile3;
+XSyscall::UnmapViewOfFile2Fn XSyscall::UnmapViewOfFile2;
+
+static void* lookup_kernelbase_library() {
+  const char* const name = "KernelBase";
+  char ebuf[1024];
+  void* const handle = os::dll_load(name, ebuf, sizeof(ebuf));
+  if (handle == nullptr) {
+    log_error_p(gc)("Failed to load library: %s", name);
+  }
+  return handle;
+}
+
+static void* lookup_kernelbase_symbol(const char* name) {
+  static void* const handle = lookup_kernelbase_library();
+  if (handle == nullptr) {
+    return nullptr;
+  }
+  return os::dll_lookup(handle, name);
+}
+
+static bool has_kernelbase_symbol(const char* name) {
+  return lookup_kernelbase_symbol(name) != nullptr;
+}
+
+template <typename Fn>
+static void install_kernelbase_symbol(Fn*& fn, const char* name) {
+  fn = reinterpret_cast<Fn*>(lookup_kernelbase_symbol(name));
+}
+
+template <typename Fn>
+static void install_kernelbase_1803_symbol_or_exit(Fn*& fn, const char* name) {
+  install_kernelbase_symbol(fn, name);
+  if (fn == nullptr) {
+    log_error_p(gc)("Failed to lookup symbol: %s", name);
+    vm_exit_during_initialization("ZGC requires Windows version 1803 or later");
+  }
+}
+
+void XSyscall::initialize() {
+  // Required
+  install_kernelbase_1803_symbol_or_exit(CreateFileMappingW, "CreateFileMappingW");
+  install_kernelbase_1803_symbol_or_exit(VirtualAlloc2,      "VirtualAlloc2");
+  install_kernelbase_1803_symbol_or_exit(VirtualFreeEx,      "VirtualFreeEx");
+  install_kernelbase_1803_symbol_or_exit(MapViewOfFile3,     "MapViewOfFile3");
+  install_kernelbase_1803_symbol_or_exit(UnmapViewOfFile2,   "UnmapViewOfFile2");
+
+  // Optional - for large pages support
+  install_kernelbase_symbol(CreateFileMapping2, "CreateFileMapping2");
+}
+
+bool XSyscall::is_supported() {
+  // Available in Windows version 1803 and later
+  return has_kernelbase_symbol("VirtualAlloc2");
+}
+
+bool XSyscall::is_large_pages_supported() {
+  // Available in Windows version 1809 and later
+  return has_kernelbase_symbol("CreateFileMapping2");
+}
diff --git a/src/hotspot/os/windows/gc/x/xSyscall_windows.hpp b/src/hotspot/os/windows/gc/x/xSyscall_windows.hpp
new file mode 100644
index 00000000000..89ba2573b10
--- /dev/null
+++ b/src/hotspot/os/windows/gc/x/xSyscall_windows.hpp
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef OS_WINDOWS_GC_X_XSYSCALL_WINDOWS_HPP
+#define OS_WINDOWS_GC_X_XSYSCALL_WINDOWS_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+#include <Windows.h>
+#include <Memoryapi.h>
+
+class XSyscall {
+private:
+  typedef HANDLE (*CreateFileMappingWFn)(HANDLE, LPSECURITY_ATTRIBUTES, DWORD, DWORD, DWORD, LPCWSTR);
+  typedef HANDLE (*CreateFileMapping2Fn)(HANDLE, LPSECURITY_ATTRIBUTES, ULONG, ULONG, ULONG, ULONG64, PCWSTR, PMEM_EXTENDED_PARAMETER, ULONG);
+  typedef PVOID (*VirtualAlloc2Fn)(HANDLE, PVOID, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER*, ULONG);
+  typedef BOOL (*VirtualFreeExFn)(HANDLE, LPVOID, SIZE_T, DWORD);
+  typedef PVOID (*MapViewOfFile3Fn)(HANDLE, HANDLE, PVOID, ULONG64, SIZE_T, ULONG, ULONG, MEM_EXTENDED_PARAMETER*, ULONG);
+  typedef BOOL (*UnmapViewOfFile2Fn)(HANDLE, PVOID, ULONG);
+
+public:
+  static CreateFileMappingWFn CreateFileMappingW;
+  static CreateFileMapping2Fn CreateFileMapping2;
+  static VirtualAlloc2Fn      VirtualAlloc2;
+  static VirtualFreeExFn      VirtualFreeEx;
+  static MapViewOfFile3Fn     MapViewOfFile3;
+  static UnmapViewOfFile2Fn   UnmapViewOfFile2;
+
+  static void initialize();
+
+  static bool is_supported();
+  static bool is_large_pages_supported();
+};
+
+#endif // OS_WINDOWS_GC_X_XSYSCALL_WINDOWS_HPP
diff --git a/src/hotspot/os/windows/gc/x/xUtils_windows.cpp b/src/hotspot/os/windows/gc/x/xUtils_windows.cpp
new file mode 100644
index 00000000000..788da80834a
--- /dev/null
+++ b/src/hotspot/os/windows/gc/x/xUtils_windows.cpp
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xUtils.hpp"
+#include "utilities/debug.hpp"
+
+#include <malloc.h>
+
+uintptr_t XUtils::alloc_aligned(size_t alignment, size_t size) {
+  void* const res = _aligned_malloc(size, alignment);
+
+  if (res == nullptr) {
+    fatal("_aligned_malloc failed");
+  }
+
+  memset(res, 0, size);
+
+  return (uintptr_t)res;
+}
diff --git a/src/hotspot/os/windows/gc/x/xVirtualMemory_windows.cpp b/src/hotspot/os/windows/gc/x/xVirtualMemory_windows.cpp
new file mode 100644
index 00000000000..c9c891430fe
--- /dev/null
+++ b/src/hotspot/os/windows/gc/x/xVirtualMemory_windows.cpp
@@ -0,0 +1,195 @@
+/*
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xLargePages.inline.hpp"
+#include "gc/x/xMapper_windows.hpp"
+#include "gc/x/xSyscall_windows.hpp"
+#include "gc/x/xVirtualMemory.hpp"
+#include "utilities/align.hpp"
+#include "utilities/debug.hpp"
+
+class XVirtualMemoryManagerImpl : public CHeapObj<mtGC> {
+public:
+  virtual void initialize_before_reserve() {}
+  virtual void initialize_after_reserve(XMemoryManager* manager) {}
+  virtual bool reserve(uintptr_t addr, size_t size) = 0;
+  virtual void unreserve(uintptr_t addr, size_t size) = 0;
+};
+
+// Implements small pages (paged) support using placeholder reservation.
+class XVirtualMemoryManagerSmallPages : public XVirtualMemoryManagerImpl {
+private:
+  class PlaceholderCallbacks : public AllStatic {
+  public:
+    static void split_placeholder(uintptr_t start, size_t size) {
+      XMapper::split_placeholder(XAddress::marked0(start), size);
+      XMapper::split_placeholder(XAddress::marked1(start), size);
+      XMapper::split_placeholder(XAddress::remapped(start), size);
+    }
+
+    static void coalesce_placeholders(uintptr_t start, size_t size) {
+      XMapper::coalesce_placeholders(XAddress::marked0(start), size);
+      XMapper::coalesce_placeholders(XAddress::marked1(start), size);
+      XMapper::coalesce_placeholders(XAddress::remapped(start), size);
+    }
+
+    static void split_into_placeholder_granules(uintptr_t start, size_t size) {
+      for (uintptr_t addr = start; addr < start + size; addr += XGranuleSize) {
+        split_placeholder(addr, XGranuleSize);
+      }
+    }
+
+    static void coalesce_into_one_placeholder(uintptr_t start, size_t size) {
+      assert(is_aligned(size, XGranuleSize), "Must be granule aligned");
+
+      if (size > XGranuleSize) {
+        coalesce_placeholders(start, size);
+      }
+    }
+
+    static void create_callback(const XMemory* area) {
+      assert(is_aligned(area->size(), XGranuleSize), "Must be granule aligned");
+      coalesce_into_one_placeholder(area->start(), area->size());
+    }
+
+    static void destroy_callback(const XMemory* area) {
+      assert(is_aligned(area->size(), XGranuleSize), "Must be granule aligned");
+      // Don't try split the last granule - VirtualFree will fail
+      split_into_placeholder_granules(area->start(), area->size() - XGranuleSize);
+    }
+
+    static void shrink_from_front_callback(const XMemory* area, size_t size) {
+      assert(is_aligned(size, XGranuleSize), "Must be granule aligned");
+      split_into_placeholder_granules(area->start(), size);
+    }
+
+    static void shrink_from_back_callback(const XMemory* area, size_t size) {
+      assert(is_aligned(size, XGranuleSize), "Must be granule aligned");
+      // Don't try split the last granule - VirtualFree will fail
+      split_into_placeholder_granules(area->end() - size, size - XGranuleSize);
+    }
+
+    static void grow_from_front_callback(const XMemory* area, size_t size) {
+      assert(is_aligned(area->size(), XGranuleSize), "Must be granule aligned");
+      coalesce_into_one_placeholder(area->start() - size, area->size() + size);
+    }
+
+    static void grow_from_back_callback(const XMemory* area, size_t size) {
+      assert(is_aligned(area->size(), XGranuleSize), "Must be granule aligned");
+      coalesce_into_one_placeholder(area->start(), area->size() + size);
+    }
+
+    static void register_with(XMemoryManager* manager) {
+      // Each reserved virtual memory address area registered in _manager is
+      // exactly covered by a single placeholder. Callbacks are installed so
+      // that whenever a memory area changes, the corresponding placeholder
+      // is adjusted.
+      //
+      // The create and grow callbacks are called when virtual memory is
+      // returned to the memory manager. The new memory area is then covered
+      // by a new single placeholder.
+      //
+      // The destroy and shrink callbacks are called when virtual memory is
+      // allocated from the memory manager. The memory area is then is split
+      // into granule-sized placeholders.
+      //
+      // See comment in zMapper_windows.cpp explaining why placeholders are
+      // split into XGranuleSize sized placeholders.
+
+      XMemoryManager::Callbacks callbacks;
+
+      callbacks._create = &create_callback;
+      callbacks._destroy = &destroy_callback;
+      callbacks._shrink_from_front = &shrink_from_front_callback;
+      callbacks._shrink_from_back = &shrink_from_back_callback;
+      callbacks._grow_from_front = &grow_from_front_callback;
+      callbacks._grow_from_back = &grow_from_back_callback;
+
+      manager->register_callbacks(callbacks);
+    }
+  };
+
+  virtual void initialize_after_reserve(XMemoryManager* manager) {
+    PlaceholderCallbacks::register_with(manager);
+  }
+
+  virtual bool reserve(uintptr_t addr, size_t size) {
+    const uintptr_t res = XMapper::reserve(addr, size);
+
+    assert(res == addr || res == 0, "Should not reserve other memory than requested");
+    return res == addr;
+  }
+
+  virtual void unreserve(uintptr_t addr, size_t size) {
+    XMapper::unreserve(addr, size);
+  }
+};
+
+// Implements Large Pages (locked) support using shared AWE physical memory.
+
+// XPhysicalMemory layer needs access to the section
+HANDLE XAWESection;
+
+class XVirtualMemoryManagerLargePages : public XVirtualMemoryManagerImpl {
+private:
+  virtual void initialize_before_reserve() {
+    XAWESection = XMapper::create_shared_awe_section();
+  }
+
+  virtual bool reserve(uintptr_t addr, size_t size) {
+    const uintptr_t res = XMapper::reserve_for_shared_awe(XAWESection, addr, size);
+
+    assert(res == addr || res == 0, "Should not reserve other memory than requested");
+    return res == addr;
+  }
+
+  virtual void unreserve(uintptr_t addr, size_t size) {
+    XMapper::unreserve_for_shared_awe(addr, size);
+  }
+};
+
+static XVirtualMemoryManagerImpl* _impl = nullptr;
+
+void XVirtualMemoryManager::pd_initialize_before_reserve() {
+  if (XLargePages::is_enabled()) {
+    _impl = new XVirtualMemoryManagerLargePages();
+  } else {
+    _impl = new XVirtualMemoryManagerSmallPages();
+  }
+  _impl->initialize_before_reserve();
+}
+
+void XVirtualMemoryManager::pd_initialize_after_reserve() {
+  _impl->initialize_after_reserve(&_manager);
+}
+
+bool XVirtualMemoryManager::pd_reserve(uintptr_t addr, size_t size) {
+  return _impl->reserve(addr, size);
+}
+
+void XVirtualMemoryManager::pd_unreserve(uintptr_t addr, size_t size) {
+  _impl->unreserve(addr, size);
+}
diff --git a/src/hotspot/os/windows/gc/z/zArguments_windows.cpp b/src/hotspot/os/windows/gc/z/zArguments_windows.cpp
index b51f4df354e..e10a06648f0 100644
--- a/src/hotspot/os/windows/gc/z/zArguments_windows.cpp
+++ b/src/hotspot/os/windows/gc/z/zArguments_windows.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,6 @@
 #include "gc/z/zArguments.hpp"
 #include "gc/z/zSyscall_windows.hpp"
 
-bool ZArguments::is_os_supported() const {
+bool ZArguments::is_os_supported() {
   return ZSyscall::is_supported();
 }
diff --git a/src/hotspot/os/windows/gc/z/zMapper_windows.cpp b/src/hotspot/os/windows/gc/z/zMapper_windows.cpp
index 95a2dc07bae..b2923a300e4 100644
--- a/src/hotspot/os/windows/gc/z/zMapper_windows.cpp
+++ b/src/hotspot/os/windows/gc/z/zMapper_windows.cpp
@@ -22,6 +22,7 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zMapper_windows.hpp"
 #include "gc/z/zSyscall_windows.hpp"
 #include "logging/log.hpp"
@@ -59,10 +60,10 @@
   fatal(msg ": " PTR_FORMAT " " SIZE_FORMAT "M (%d)", \
         (addr), (size) / M, GetLastError())
 
-uintptr_t ZMapper::reserve(uintptr_t addr, size_t size) {
+zaddress_unsafe ZMapper::reserve(zaddress_unsafe addr, size_t size) {
   void* const res = ZSyscall::VirtualAlloc2(
     GetCurrentProcess(),                   // Process
-    (void*)addr,                           // BaseAddress
+    (void*)untype(addr),                   // BaseAddress
     size,                                  // Size
     MEM_RESERVE | MEM_RESERVE_PLACEHOLDER, // AllocationType
     PAGE_NOACCESS,                         // PageProtection
@@ -71,19 +72,19 @@ uintptr_t ZMapper::reserve(uintptr_t addr, size_t size) {
     );
 
   // Caller responsible for error handling
-  return (uintptr_t)res;
+  return to_zaddress_unsafe((uintptr_t)res);
 }
 
-void ZMapper::unreserve(uintptr_t addr, size_t size) {
+void ZMapper::unreserve(zaddress_unsafe addr, size_t size) {
   const bool res = ZSyscall::VirtualFreeEx(
     GetCurrentProcess(), // hProcess
-    (void*)addr,         // lpAddress
+    (void*)untype(addr), // lpAddress
     size,                // dwSize
     MEM_RELEASE          // dwFreeType
     );
 
   if (!res) {
-    fatal_error("Failed to unreserve memory", addr, size);
+    fatal_error("Failed to unreserve memory", untype(addr), size);
   }
 }
 
@@ -223,14 +224,14 @@ HANDLE ZMapper::create_shared_awe_section() {
   return section;
 }
 
-uintptr_t ZMapper::reserve_for_shared_awe(HANDLE awe_section, uintptr_t addr, size_t size) {
+zaddress_unsafe ZMapper::reserve_for_shared_awe(HANDLE awe_section, zaddress_unsafe addr, size_t size) {
   MEM_EXTENDED_PARAMETER parameter = { 0 };
   parameter.Type = MemExtendedParameterUserPhysicalHandle;
   parameter.Handle = awe_section;
 
   void* const res = ZSyscall::VirtualAlloc2(
     GetCurrentProcess(),        // Process
-    (void*)addr,                // BaseAddress
+    (void*)untype(addr),        // BaseAddress
     size,                       // Size
     MEM_RESERVE | MEM_PHYSICAL, // AllocationType
     PAGE_READWRITE,             // PageProtection
@@ -239,25 +240,25 @@ uintptr_t ZMapper::reserve_for_shared_awe(HANDLE awe_section, uintptr_t addr, si
     );
 
   // Caller responsible for error handling
-  return (uintptr_t)res;
+  return to_zaddress_unsafe((uintptr_t)res);
 }
 
-void ZMapper::unreserve_for_shared_awe(uintptr_t addr, size_t size) {
+void ZMapper::unreserve_for_shared_awe(zaddress_unsafe addr, size_t size) {
   bool res = VirtualFree(
-    (void*)addr, // lpAddress
-    0,           // dwSize
-    MEM_RELEASE  // dwFreeType
+    (void*)untype(addr), // lpAddress
+    0,                   // dwSize
+    MEM_RELEASE          // dwFreeType
     );
 
   if (!res) {
     fatal("Failed to unreserve memory: " PTR_FORMAT " " SIZE_FORMAT "M (%d)",
-          addr, size / M, GetLastError());
+          untype(addr), size / M, GetLastError());
   }
 }
 
-void ZMapper::split_placeholder(uintptr_t addr, size_t size) {
+void ZMapper::split_placeholder(zaddress_unsafe addr, size_t size) {
   const bool res = VirtualFree(
-    (void*)addr,                           // lpAddress
+    (void*)untype(addr),                   // lpAddress
     size,                                  // dwSize
     MEM_RELEASE | MEM_PRESERVE_PLACEHOLDER // dwFreeType
     );
@@ -267,9 +268,9 @@ void ZMapper::split_placeholder(uintptr_t addr, size_t size) {
   }
 }
 
-void ZMapper::coalesce_placeholders(uintptr_t addr, size_t size) {
+void ZMapper::coalesce_placeholders(zaddress_unsafe addr, size_t size) {
   const bool res = VirtualFree(
-    (void*)addr,                            // lpAddress
+    (void*)untype(addr),                    // lpAddress
     size,                                   // dwSize
     MEM_RELEASE | MEM_COALESCE_PLACEHOLDERS // dwFreeType
     );
@@ -279,11 +280,11 @@ void ZMapper::coalesce_placeholders(uintptr_t addr, size_t size) {
   }
 }
 
-void ZMapper::map_view_replace_placeholder(HANDLE file_handle, uintptr_t file_offset, uintptr_t addr, size_t size) {
+void ZMapper::map_view_replace_placeholder(HANDLE file_handle, uintptr_t file_offset, zaddress_unsafe addr, size_t size) {
   void* const res = ZSyscall::MapViewOfFile3(
     file_handle,             // FileMapping
     GetCurrentProcess(),     // ProcessHandle
-    (void*)addr,             // BaseAddress
+    (void*)untype(addr),     // BaseAddress
     file_offset,             // Offset
     size,                    // ViewSize
     MEM_REPLACE_PLACEHOLDER, // AllocationType
@@ -297,10 +298,10 @@ void ZMapper::map_view_replace_placeholder(HANDLE file_handle, uintptr_t file_of
   }
 }
 
-void ZMapper::unmap_view_preserve_placeholder(uintptr_t addr, size_t size) {
+void ZMapper::unmap_view_preserve_placeholder(zaddress_unsafe addr, size_t size) {
   const bool res = ZSyscall::UnmapViewOfFile2(
     GetCurrentProcess(),     // ProcessHandle
-    (void*)addr,             // BaseAddress
+    (void*)untype(addr),     // BaseAddress
     MEM_PRESERVE_PLACEHOLDER // UnmapFlags
     );
 
diff --git a/src/hotspot/os/windows/gc/z/zMapper_windows.hpp b/src/hotspot/os/windows/gc/z/zMapper_windows.hpp
index 3e47b470f5f..e6efd6b9e11 100644
--- a/src/hotspot/os/windows/gc/z/zMapper_windows.hpp
+++ b/src/hotspot/os/windows/gc/z/zMapper_windows.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
 #ifndef OS_WINDOWS_GC_Z_ZMAPPER_WINDOWS_HPP
 #define OS_WINDOWS_GC_Z_ZMAPPER_WINDOWS_HPP
 
+#include "gc/z/zAddress.hpp"
 #include "memory/allStatic.hpp"
 #include "utilities/globalDefinitions.hpp"
 
@@ -48,10 +49,10 @@ private:
 
 public:
   // Reserve memory with a placeholder
-  static uintptr_t reserve(uintptr_t addr, size_t size);
+  static zaddress_unsafe reserve(zaddress_unsafe addr, size_t size);
 
   // Unreserve memory
-  static void unreserve(uintptr_t addr, size_t size);
+  static void unreserve(zaddress_unsafe addr, size_t size);
 
   // Create and commit paging file mapping
   static HANDLE create_and_commit_paging_file_mapping(size_t size);
@@ -63,10 +64,10 @@ public:
   static HANDLE create_shared_awe_section();
 
   // Reserve memory attached to the shared AWE section
-  static uintptr_t reserve_for_shared_awe(HANDLE awe_section, uintptr_t addr, size_t size);
+  static zaddress_unsafe reserve_for_shared_awe(HANDLE awe_section, zaddress_unsafe addr, size_t size);
 
   // Unreserve memory attached to a shared AWE section
-  static void unreserve_for_shared_awe(uintptr_t addr, size_t size);
+  static void unreserve_for_shared_awe(zaddress_unsafe addr, size_t size);
 
   // Split a placeholder
   //
@@ -74,21 +75,21 @@ public:
   // split and coalesced to be the exact size of the new views.
   // [addr, addr + size) needs to be a proper sub-placeholder of an existing
   // placeholder.
-  static void split_placeholder(uintptr_t addr, size_t size);
+  static void split_placeholder(zaddress_unsafe addr, size_t size);
 
   // Coalesce a placeholder
   //
   // [addr, addr + size) is the new placeholder. A sub-placeholder needs to
   // exist within that range.
-  static void coalesce_placeholders(uintptr_t addr, size_t size);
+  static void coalesce_placeholders(zaddress_unsafe addr, size_t size);
 
   // Map a view of the file handle and replace the placeholder covering the
   // given virtual address range
-  static void map_view_replace_placeholder(HANDLE file_handle, uintptr_t file_offset, uintptr_t addr, size_t size);
+  static void map_view_replace_placeholder(HANDLE file_handle, uintptr_t file_offset, zaddress_unsafe addr, size_t size);
 
   // Unmap the view and reinstate a placeholder covering the given virtual
   // address range
-  static void unmap_view_preserve_placeholder(uintptr_t addr, size_t size);
+  static void unmap_view_preserve_placeholder(zaddress_unsafe addr, size_t size);
 };
 
 #endif // OS_WINDOWS_GC_Z_ZMAPPER_WINDOWS_HPP
diff --git a/src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.cpp b/src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.cpp
index fc7105ad8a3..3fed88f7218 100644
--- a/src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.cpp
+++ b/src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.cpp
@@ -22,6 +22,7 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zGranuleMap.inline.hpp"
 #include "gc/z/zLargePages.inline.hpp"
@@ -33,10 +34,10 @@
 
 class ZPhysicalMemoryBackingImpl : public CHeapObj<mtGC> {
 public:
-  virtual size_t commit(size_t offset, size_t size) = 0;
-  virtual size_t uncommit(size_t offset, size_t size) = 0;
-  virtual void map(uintptr_t addr, size_t size, size_t offset) const = 0;
-  virtual void unmap(uintptr_t addr, size_t size) const = 0;
+  virtual size_t commit(zoffset offset, size_t size) = 0;
+  virtual size_t uncommit(zoffset offset, size_t size) = 0;
+  virtual void map(zaddress_unsafe addr, size_t size, zoffset offset) const = 0;
+  virtual void unmap(zaddress_unsafe addr, size_t size) const = 0;
 };
 
 // Implements small pages (paged) support using placeholder reservation.
@@ -50,19 +51,19 @@ class ZPhysicalMemoryBackingSmallPages : public ZPhysicalMemoryBackingImpl {
 private:
   ZGranuleMap<HANDLE> _handles;
 
-  HANDLE get_handle(uintptr_t offset) const {
+  HANDLE get_handle(zoffset offset) const {
     HANDLE const handle = _handles.get(offset);
     assert(handle != 0, "Should be set");
     return handle;
   }
 
-  void put_handle(uintptr_t offset, HANDLE handle) {
+  void put_handle(zoffset offset, HANDLE handle) {
     assert(handle != INVALID_HANDLE_VALUE, "Invalid handle");
     assert(_handles.get(offset) == 0, "Should be cleared");
     _handles.put(offset, handle);
   }
 
-  void clear_handle(uintptr_t offset) {
+  void clear_handle(zoffset offset) {
     assert(_handles.get(offset) != 0, "Should be set");
     _handles.put(offset, 0);
   }
@@ -72,7 +73,7 @@ public:
       ZPhysicalMemoryBackingImpl(),
       _handles(max_capacity) {}
 
-  size_t commit(size_t offset, size_t size) {
+  size_t commit(zoffset offset, size_t size) {
     for (size_t i = 0; i < size; i += ZGranuleSize) {
       HANDLE const handle = ZMapper::create_and_commit_paging_file_mapping(ZGranuleSize);
       if (handle == 0) {
@@ -85,7 +86,7 @@ public:
     return size;
   }
 
-  size_t uncommit(size_t offset, size_t size) {
+  size_t uncommit(zoffset offset, size_t size) {
     for (size_t i = 0; i < size; i += ZGranuleSize) {
       HANDLE const handle = get_handle(offset + i);
       clear_handle(offset + i);
@@ -95,9 +96,9 @@ public:
     return size;
   }
 
-  void map(uintptr_t addr, size_t size, size_t offset) const {
-    assert(is_aligned(offset, ZGranuleSize), "Misaligned");
-    assert(is_aligned(addr, ZGranuleSize), "Misaligned");
+  void map(zaddress_unsafe addr, size_t size, zoffset offset) const {
+    assert(is_aligned(untype(offset), ZGranuleSize), "Misaligned");
+    assert(is_aligned(untype(addr), ZGranuleSize), "Misaligned");
     assert(is_aligned(size, ZGranuleSize), "Misaligned");
 
     for (size_t i = 0; i < size; i += ZGranuleSize) {
@@ -106,12 +107,12 @@ public:
     }
   }
 
-  void unmap(uintptr_t addr, size_t size) const {
-    assert(is_aligned(addr, ZGranuleSize), "Misaligned");
+  void unmap(zaddress_unsafe addr, size_t size) const {
+    assert(is_aligned(untype(addr), ZGranuleSize), "Misaligned");
     assert(is_aligned(size, ZGranuleSize), "Misaligned");
 
     for (size_t i = 0; i < size; i += ZGranuleSize) {
-      ZMapper::unmap_view_preserve_placeholder(addr + i, ZGranuleSize);
+      ZMapper::unmap_view_preserve_placeholder(to_zaddress_unsafe(untype(addr) + i), ZGranuleSize);
     }
   }
 };
@@ -149,17 +150,17 @@ public:
       ZPhysicalMemoryBackingImpl(),
       _page_array(alloc_page_array(max_capacity)) {}
 
-  size_t commit(size_t offset, size_t size) {
-    const size_t index = offset >> ZGranuleSizeShift;
+  size_t commit(zoffset offset, size_t size) {
+    const size_t index = untype(offset) >> ZGranuleSizeShift;
     const size_t npages = size >> ZGranuleSizeShift;
 
     size_t npages_res = npages;
     const bool res = AllocateUserPhysicalPages(ZAWESection, &npages_res, &_page_array[index]);
     if (!res) {
       fatal("Failed to allocate physical memory " SIZE_FORMAT "M @ " PTR_FORMAT " (%d)",
-            size / M, offset, GetLastError());
+            size / M, untype(offset), GetLastError());
     } else {
-      log_debug(gc)("Allocated physical memory: " SIZE_FORMAT "M @ " PTR_FORMAT, size / M, offset);
+      log_debug(gc)("Allocated physical memory: " SIZE_FORMAT "M @ " PTR_FORMAT, size / M, untype(offset));
     }
 
     // AllocateUserPhysicalPages might not be able to allocate the requested amount of memory.
@@ -167,35 +168,35 @@ public:
     return npages_res << ZGranuleSizeShift;
   }
 
-  size_t uncommit(size_t offset, size_t size) {
-    const size_t index = offset >> ZGranuleSizeShift;
+  size_t uncommit(zoffset offset, size_t size) {
+    const size_t index = untype(offset) >> ZGranuleSizeShift;
     const size_t npages = size >> ZGranuleSizeShift;
 
     size_t npages_res = npages;
     const bool res = FreeUserPhysicalPages(ZAWESection, &npages_res, &_page_array[index]);
     if (!res) {
       fatal("Failed to uncommit physical memory " SIZE_FORMAT "M @ " PTR_FORMAT " (%d)",
-            size, offset, GetLastError());
+            size, untype(offset), GetLastError());
     }
 
     return npages_res << ZGranuleSizeShift;
   }
 
-  void map(uintptr_t addr, size_t size, size_t offset) const {
+  void map(zaddress_unsafe addr, size_t size, zoffset offset) const {
     const size_t npages = size >> ZGranuleSizeShift;
-    const size_t index = offset >> ZGranuleSizeShift;
+    const size_t index = untype(offset) >> ZGranuleSizeShift;
 
-    const bool res = MapUserPhysicalPages((char*)addr, npages, &_page_array[index]);
+    const bool res = MapUserPhysicalPages((char*)untype(addr), npages, &_page_array[index]);
     if (!res) {
       fatal("Failed to map view " PTR_FORMAT " " SIZE_FORMAT "M @ " PTR_FORMAT " (%d)",
-            addr, size / M, offset, GetLastError());
+            untype(addr), size / M, untype(offset), GetLastError());
     }
   }
 
-  void unmap(uintptr_t addr, size_t size) const {
+  void unmap(zaddress_unsafe addr, size_t size) const {
     const size_t npages = size >> ZGranuleSizeShift;
 
-    const bool res = MapUserPhysicalPages((char*)addr, npages, nullptr);
+    const bool res = MapUserPhysicalPages((char*)untype(addr), npages, nullptr);
     if (!res) {
       fatal("Failed to unmap view " PTR_FORMAT " " SIZE_FORMAT "M (%d)",
             addr, size / M, GetLastError());
@@ -222,30 +223,30 @@ void ZPhysicalMemoryBacking::warn_commit_limits(size_t max_capacity) const {
   // Does nothing
 }
 
-size_t ZPhysicalMemoryBacking::commit(size_t offset, size_t length) {
+size_t ZPhysicalMemoryBacking::commit(zoffset offset, size_t length) {
   log_trace(gc, heap)("Committing memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
-                      offset / M, (offset + length) / M, length / M);
+                      untype(offset) / M, (untype(offset) + length) / M, length / M);
 
   return _impl->commit(offset, length);
 }
 
-size_t ZPhysicalMemoryBacking::uncommit(size_t offset, size_t length) {
+size_t ZPhysicalMemoryBacking::uncommit(zoffset offset, size_t length) {
   log_trace(gc, heap)("Uncommitting memory: " SIZE_FORMAT "M-" SIZE_FORMAT "M (" SIZE_FORMAT "M)",
-                      offset / M, (offset + length) / M, length / M);
+                      untype(offset) / M, (untype(offset) + length) / M, length / M);
 
   return _impl->uncommit(offset, length);
 }
 
-void ZPhysicalMemoryBacking::map(uintptr_t addr, size_t size, size_t offset) const {
-  assert(is_aligned(offset, ZGranuleSize), "Misaligned: " PTR_FORMAT, offset);
-  assert(is_aligned(addr, ZGranuleSize), "Misaligned: " PTR_FORMAT, addr);
+void ZPhysicalMemoryBacking::map(zaddress_unsafe addr, size_t size, zoffset offset) const {
+  assert(is_aligned(untype(offset), ZGranuleSize), "Misaligned: " PTR_FORMAT, untype(offset));
+  assert(is_aligned(untype(addr), ZGranuleSize), "Misaligned: " PTR_FORMAT, addr);
   assert(is_aligned(size, ZGranuleSize), "Misaligned: " PTR_FORMAT, size);
 
   _impl->map(addr, size, offset);
 }
 
-void ZPhysicalMemoryBacking::unmap(uintptr_t addr, size_t size) const {
-  assert(is_aligned(addr, ZGranuleSize), "Misaligned");
+void ZPhysicalMemoryBacking::unmap(zaddress_unsafe addr, size_t size) const {
+  assert(is_aligned(untype(addr), ZGranuleSize), "Misaligned");
   assert(is_aligned(size, ZGranuleSize), "Misaligned");
 
   _impl->unmap(addr, size);
diff --git a/src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.hpp b/src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.hpp
index 3d3479c2a1a..b8b73519ab5 100644
--- a/src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.hpp
+++ b/src/hotspot/os/windows/gc/z/zPhysicalMemoryBacking_windows.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
 #ifndef OS_WINDOWS_GC_Z_ZPHYSICALMEMORYBACKING_WINDOWS_HPP
 #define OS_WINDOWS_GC_Z_ZPHYSICALMEMORYBACKING_WINDOWS_HPP
 
+#include "gc/z/zAddress.hpp"
 #include "utilities/globalDefinitions.hpp"
 
 #include <Windows.h>
@@ -41,11 +42,11 @@ public:
 
   void warn_commit_limits(size_t max_capacity) const;
 
-  size_t commit(size_t offset, size_t length);
-  size_t uncommit(size_t offset, size_t length);
+  size_t commit(zoffset offset, size_t length);
+  size_t uncommit(zoffset offset, size_t length);
 
-  void map(uintptr_t addr, size_t size, size_t offset) const;
-  void unmap(uintptr_t addr, size_t size) const;
+  void map(zaddress_unsafe addr, size_t size, zoffset offset) const;
+  void unmap(zaddress_unsafe addr, size_t size) const;
 };
 
 #endif // OS_WINDOWS_GC_Z_ZPHYSICALMEMORYBACKING_WINDOWS_HPP
diff --git a/src/hotspot/os/windows/gc/z/zVirtualMemory_windows.cpp b/src/hotspot/os/windows/gc/z/zVirtualMemory_windows.cpp
index de31b49d940..581296a5ec3 100644
--- a/src/hotspot/os/windows/gc/z/zVirtualMemory_windows.cpp
+++ b/src/hotspot/os/windows/gc/z/zVirtualMemory_windows.cpp
@@ -35,8 +35,8 @@ class ZVirtualMemoryManagerImpl : public CHeapObj<mtGC> {
 public:
   virtual void initialize_before_reserve() {}
   virtual void initialize_after_reserve(ZMemoryManager* manager) {}
-  virtual bool reserve(uintptr_t addr, size_t size) = 0;
-  virtual void unreserve(uintptr_t addr, size_t size) = 0;
+  virtual bool reserve(zaddress_unsafe addr, size_t size) = 0;
+  virtual void unreserve(zaddress_unsafe addr, size_t size) = 0;
 };
 
 // Implements small pages (paged) support using placeholder reservation.
@@ -44,25 +44,21 @@ class ZVirtualMemoryManagerSmallPages : public ZVirtualMemoryManagerImpl {
 private:
   class PlaceholderCallbacks : public AllStatic {
   public:
-    static void split_placeholder(uintptr_t start, size_t size) {
-      ZMapper::split_placeholder(ZAddress::marked0(start), size);
-      ZMapper::split_placeholder(ZAddress::marked1(start), size);
-      ZMapper::split_placeholder(ZAddress::remapped(start), size);
+    static void split_placeholder(zoffset start, size_t size) {
+      ZMapper::split_placeholder(ZOffset::address_unsafe(start), size);
     }
 
-    static void coalesce_placeholders(uintptr_t start, size_t size) {
-      ZMapper::coalesce_placeholders(ZAddress::marked0(start), size);
-      ZMapper::coalesce_placeholders(ZAddress::marked1(start), size);
-      ZMapper::coalesce_placeholders(ZAddress::remapped(start), size);
+    static void coalesce_placeholders(zoffset start, size_t size) {
+      ZMapper::coalesce_placeholders(ZOffset::address_unsafe(start), size);
     }
 
-    static void split_into_placeholder_granules(uintptr_t start, size_t size) {
-      for (uintptr_t addr = start; addr < start + size; addr += ZGranuleSize) {
-        split_placeholder(addr, ZGranuleSize);
+    static void split_into_placeholder_granules(zoffset start, size_t size) {
+      for (uintptr_t addr = untype(start); addr < untype(start) + size; addr += ZGranuleSize) {
+        split_placeholder(to_zoffset(addr), ZGranuleSize);
       }
     }
 
-    static void coalesce_into_one_placeholder(uintptr_t start, size_t size) {
+    static void coalesce_into_one_placeholder(zoffset start, size_t size) {
       assert(is_aligned(size, ZGranuleSize), "Must be granule aligned");
 
       if (size > ZGranuleSize) {
@@ -89,12 +85,12 @@ private:
     static void shrink_from_back_callback(const ZMemory* area, size_t size) {
       assert(is_aligned(size, ZGranuleSize), "Must be granule aligned");
       // Don't try split the last granule - VirtualFree will fail
-      split_into_placeholder_granules(area->end() - size, size - ZGranuleSize);
+      split_into_placeholder_granules(to_zoffset(untype(area->end()) - size), size - ZGranuleSize);
     }
 
     static void grow_from_front_callback(const ZMemory* area, size_t size) {
       assert(is_aligned(area->size(), ZGranuleSize), "Must be granule aligned");
-      coalesce_into_one_placeholder(area->start() - size, area->size() + size);
+      coalesce_into_one_placeholder(to_zoffset(untype(area->start()) - size), area->size() + size);
     }
 
     static void grow_from_back_callback(const ZMemory* area, size_t size) {
@@ -136,14 +132,14 @@ private:
     PlaceholderCallbacks::register_with(manager);
   }
 
-  virtual bool reserve(uintptr_t addr, size_t size) {
-    const uintptr_t res = ZMapper::reserve(addr, size);
+  virtual bool reserve(zaddress_unsafe addr, size_t size) {
+    const zaddress_unsafe res = ZMapper::reserve(addr, size);
 
-    assert(res == addr || res == 0, "Should not reserve other memory than requested");
+    assert(res == addr || untype(res) == 0, "Should not reserve other memory than requested");
     return res == addr;
   }
 
-  virtual void unreserve(uintptr_t addr, size_t size) {
+  virtual void unreserve(zaddress_unsafe addr, size_t size) {
     ZMapper::unreserve(addr, size);
   }
 };
@@ -159,14 +155,14 @@ private:
     ZAWESection = ZMapper::create_shared_awe_section();
   }
 
-  virtual bool reserve(uintptr_t addr, size_t size) {
-    const uintptr_t res = ZMapper::reserve_for_shared_awe(ZAWESection, addr, size);
+  virtual bool reserve(zaddress_unsafe addr, size_t size) {
+    const zaddress_unsafe res = ZMapper::reserve_for_shared_awe(ZAWESection, addr, size);
 
-    assert(res == addr || res == 0, "Should not reserve other memory than requested");
+    assert(res == addr || untype(res) == 0, "Should not reserve other memory than requested");
     return res == addr;
   }
 
-  virtual void unreserve(uintptr_t addr, size_t size) {
+  virtual void unreserve(zaddress_unsafe addr, size_t size) {
     ZMapper::unreserve_for_shared_awe(addr, size);
   }
 };
@@ -186,10 +182,10 @@ void ZVirtualMemoryManager::pd_initialize_after_reserve() {
   _impl->initialize_after_reserve(&_manager);
 }
 
-bool ZVirtualMemoryManager::pd_reserve(uintptr_t addr, size_t size) {
+bool ZVirtualMemoryManager::pd_reserve(zaddress_unsafe addr, size_t size) {
   return _impl->reserve(addr, size);
 }
 
-void ZVirtualMemoryManager::pd_unreserve(uintptr_t addr, size_t size) {
+void ZVirtualMemoryManager::pd_unreserve(zaddress_unsafe addr, size_t size) {
   _impl->unreserve(addr, size);
 }
diff --git a/src/hotspot/os_cpu/bsd_aarch64/atomic_bsd_aarch64.hpp b/src/hotspot/os_cpu/bsd_aarch64/atomic_bsd_aarch64.hpp
index 5a88b1a32f0..53759332df7 100644
--- a/src/hotspot/os_cpu/bsd_aarch64/atomic_bsd_aarch64.hpp
+++ b/src/hotspot/os_cpu/bsd_aarch64/atomic_bsd_aarch64.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2014, 2019, Red Hat Inc. All rights reserved.
  * Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
diff --git a/src/hotspot/os_cpu/linux_aarch64/gc/x/xSyscall_linux_aarch64.hpp b/src/hotspot/os_cpu/linux_aarch64/gc/x/xSyscall_linux_aarch64.hpp
new file mode 100644
index 00000000000..b4c49f477a6
--- /dev/null
+++ b/src/hotspot/os_cpu/linux_aarch64/gc/x/xSyscall_linux_aarch64.hpp
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef OS_CPU_LINUX_AARCH64_GC_X_XSYSCALL_LINUX_AARCH64_HPP
+#define OS_CPU_LINUX_AARCH64_GC_X_XSYSCALL_LINUX_AARCH64_HPP
+
+#include <sys/syscall.h>
+
+//
+// Support for building on older Linux systems
+//
+
+#ifndef SYS_memfd_create
+#define SYS_memfd_create     279
+#endif
+#ifndef SYS_fallocate
+#define SYS_fallocate        47
+#endif
+
+#endif // OS_CPU_LINUX_AARCH64_GC_X_XSYSCALL_LINUX_AARCH64_HPP
diff --git a/src/hotspot/os_cpu/linux_ppc/gc/x/xSyscall_linux_ppc.hpp b/src/hotspot/os_cpu/linux_ppc/gc/x/xSyscall_linux_ppc.hpp
new file mode 100644
index 00000000000..22d51cd58f5
--- /dev/null
+++ b/src/hotspot/os_cpu/linux_ppc/gc/x/xSyscall_linux_ppc.hpp
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021 SAP SE. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef OS_CPU_LINUX_PPC_GC_X_XSYSCALL_LINUX_PPC_HPP
+#define OS_CPU_LINUX_PPC_GC_X_XSYSCALL_LINUX_PPC_HPP
+
+#include <sys/syscall.h>
+
+//
+// Support for building on older Linux systems
+//
+
+
+#ifndef SYS_memfd_create
+#define SYS_memfd_create     360
+#endif
+#ifndef SYS_fallocate
+#define SYS_fallocate        309
+#endif
+
+#endif // OS_CPU_LINUX_PPC_GC_X_XSYSCALL_LINUX_PPC_HPP
diff --git a/src/hotspot/os_cpu/linux_riscv/gc/x/xSyscall_linux_riscv.hpp b/src/hotspot/os_cpu/linux_riscv/gc/x/xSyscall_linux_riscv.hpp
new file mode 100644
index 00000000000..bfd49b0bf4e
--- /dev/null
+++ b/src/hotspot/os_cpu/linux_riscv/gc/x/xSyscall_linux_riscv.hpp
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef OS_CPU_LINUX_RISCV_GC_X_XSYSCALL_LINUX_RISCV_HPP
+#define OS_CPU_LINUX_RISCV_GC_X_XSYSCALL_LINUX_RISCV_HPP
+
+#include <sys/syscall.h>
+
+//
+// Support for building on older Linux systems
+//
+
+#ifndef SYS_memfd_create
+#define SYS_memfd_create     279
+#endif
+#ifndef SYS_fallocate
+#define SYS_fallocate        47
+#endif
+
+#endif // OS_CPU_LINUX_RISCV_GC_X_XSYSCALL_LINUX_RISCV_HPP
diff --git a/src/hotspot/os_cpu/linux_x86/gc/x/xSyscall_linux_x86.hpp b/src/hotspot/os_cpu/linux_x86/gc/x/xSyscall_linux_x86.hpp
new file mode 100644
index 00000000000..2709b373b28
--- /dev/null
+++ b/src/hotspot/os_cpu/linux_x86/gc/x/xSyscall_linux_x86.hpp
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef OS_CPU_LINUX_X86_GC_X_XSYSCALL_LINUX_X86_HPP
+#define OS_CPU_LINUX_X86_GC_X_XSYSCALL_LINUX_X86_HPP
+
+#include <sys/syscall.h>
+
+//
+// Support for building on older Linux systems
+//
+
+#ifndef SYS_memfd_create
+#define SYS_memfd_create     319
+#endif
+#ifndef SYS_fallocate
+#define SYS_fallocate        285
+#endif
+
+#endif // OS_CPU_LINUX_X86_GC_X_XSYSCALL_LINUX_X86_HPP
diff --git a/src/hotspot/share/asm/assembler.hpp b/src/hotspot/share/asm/assembler.hpp
index 1593d388579..7629d368241 100644
--- a/src/hotspot/share/asm/assembler.hpp
+++ b/src/hotspot/share/asm/assembler.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -242,6 +242,7 @@ class AbstractAssembler : public ResourceObj  {
   };
   friend class InstructionMark;
 
+ public:
   // count size of instructions which are skipped from inline heuristics
   class InlineSkippedInstructionsCounter: public StackObj {
    private:
@@ -254,6 +255,8 @@ class AbstractAssembler : public ResourceObj  {
       _assm->register_skipped(_assm->pc() - _start);
     }
   };
+
+ protected:
 #ifdef ASSERT
   // Make it return true on platforms which need to verify
   // instruction boundaries for some operations.
diff --git a/src/hotspot/share/asm/codeBuffer.cpp b/src/hotspot/share/asm/codeBuffer.cpp
index 8c8230bd9ae..95effc5c5af 100644
--- a/src/hotspot/share/asm/codeBuffer.cpp
+++ b/src/hotspot/share/asm/codeBuffer.cpp
@@ -323,7 +323,8 @@ void CodeSection::relocate(address at, RelocationHolder const& spec, int format)
            rtype == relocInfo::runtime_call_type ||
            rtype == relocInfo::internal_word_type||
            rtype == relocInfo::section_word_type ||
-           rtype == relocInfo::external_word_type,
+           rtype == relocInfo::external_word_type||
+           rtype == relocInfo::barrier_type,
            "code needs relocation information");
     // leave behind an indication that we attempted a relocation
     DEBUG_ONLY(_locs_start = _locs_limit = (relocInfo*)badAddress);
diff --git a/src/hotspot/share/asm/codeBuffer.hpp b/src/hotspot/share/asm/codeBuffer.hpp
index 029691c95fd..b4e41ba5178 100644
--- a/src/hotspot/share/asm/codeBuffer.hpp
+++ b/src/hotspot/share/asm/codeBuffer.hpp
@@ -180,6 +180,7 @@ class CodeSection {
 
   // Mark scratch buffer.
   void        set_scratch_emit()    { _scratch_emit = true; }
+  void        clear_scratch_emit()  { _scratch_emit = false; }
   bool        scratch_emit()        { return _scratch_emit; }
 
   CodeBuffer* outer() const         { return _outer; }
diff --git a/src/hotspot/share/c1/c1_LIR.cpp b/src/hotspot/share/c1/c1_LIR.cpp
index a553af52a80..bddc4c9cbbc 100644
--- a/src/hotspot/share/c1/c1_LIR.cpp
+++ b/src/hotspot/share/c1/c1_LIR.cpp
@@ -1143,8 +1143,14 @@ void LIR_List::set_cmp_oprs(LIR_Op* op) {
       op->as_Op4()->set_in_opr3(_cmp_opr1);
       op->as_Op4()->set_in_opr4(_cmp_opr2);
       break;
+    case lir_cas_long:
+    case lir_cas_obj:
+    case lir_cas_int:
+      _cmp_opr1 = op->as_OpCompareAndSwap()->result_opr();
+      _cmp_opr2 = LIR_OprFact::intConst(0);
+      break;
 #if INCLUDE_ZGC
-    case lir_zloadbarrier_test:
+    case lir_xloadbarrier_test:
       _cmp_opr1 = FrameMap::as_opr(t1);
       _cmp_opr2 = LIR_OprFact::intConst(0);
       break;
diff --git a/src/hotspot/share/c1/c1_LIR.hpp b/src/hotspot/share/c1/c1_LIR.hpp
index a0987a23feb..90f8bbb2d5b 100644
--- a/src/hotspot/share/c1/c1_LIR.hpp
+++ b/src/hotspot/share/c1/c1_LIR.hpp
@@ -1018,9 +1018,9 @@ enum LIR_Code {
     , lir_assert
   , end_opAssert
 #ifdef INCLUDE_ZGC
-  , begin_opZLoadBarrierTest
-    , lir_zloadbarrier_test
-  , end_opZLoadBarrierTest
+  , begin_opXLoadBarrierTest
+    , lir_xloadbarrier_test
+  , end_opXLoadBarrierTest
 #endif
 };
 
diff --git a/src/hotspot/share/c1/c1_LIRAssembler.hpp b/src/hotspot/share/c1/c1_LIRAssembler.hpp
index 270a63d7cfc..8c89cb0adfa 100644
--- a/src/hotspot/share/c1/c1_LIRAssembler.hpp
+++ b/src/hotspot/share/c1/c1_LIRAssembler.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -91,6 +91,7 @@ class LIR_Assembler: public CompilationResourceObj {
 
   void emit_stubs(CodeStubList* stub_list);
 
+ public:
   // addresses
   Address as_Address(LIR_Address* addr);
   Address as_Address_lo(LIR_Address* addr);
@@ -104,6 +105,7 @@ class LIR_Assembler: public CompilationResourceObj {
   ImplicitNullCheckStub* add_debug_info_for_null_check(int pc_offset, CodeEmitInfo* cinfo);
   ImplicitNullCheckStub* add_debug_info_for_null_check_here(CodeEmitInfo* info);
 
+ private:
   void breakpoint();
   void push(LIR_Opr opr);
   void pop(LIR_Opr opr);
diff --git a/src/hotspot/share/code/relocInfo.hpp b/src/hotspot/share/code/relocInfo.hpp
index ef8027760d7..2972f30ce56 100644
--- a/src/hotspot/share/code/relocInfo.hpp
+++ b/src/hotspot/share/code/relocInfo.hpp
@@ -275,6 +275,7 @@ class relocInfo {
     data_prefix_tag         = 15, // tag for a prefix (carries data arguments)
     post_call_nop_type      = 16, // A tag for post call nop relocations
     entry_guard_type        = 17, // A tag for an nmethod entry barrier guard value
+    barrier_type            = 18, // GC barrier data
     type_mask               = 31  // A mask which selects only the above values
   };
 
@@ -316,6 +317,7 @@ class relocInfo {
     visitor(trampoline_stub) \
     visitor(post_call_nop) \
     visitor(entry_guard) \
+    visitor(barrier) \
 
 
  public:
@@ -829,7 +831,6 @@ class Relocation {
  protected:
   short*   data()         const { return binding()->data(); }
   int      datalen()      const { return binding()->datalen(); }
-  int      format()       const { return binding()->format(); }
 
  public:
   // Make a filler relocation.
@@ -841,6 +842,8 @@ class Relocation {
   // trivial, so this must not be virtual (and hence non-trivial).
   ~Relocation() = default;
 
+  int      format()       const { return binding()->format(); }
+
   relocInfo::relocType type()              const { return _rtype; }
 
   // Copy this relocation into holder.
@@ -1078,6 +1081,26 @@ class metadata_Relocation : public DataRelocation {
 };
 
 
+class barrier_Relocation : public Relocation {
+
+ public:
+  // The uninitialized value used before the relocation has been patched.
+  // Code assumes that the unpatched value is zero.
+  static const int16_t unpatched = 0;
+
+  static RelocationHolder spec() {
+    return RelocationHolder::construct<barrier_Relocation>();
+  }
+
+  void copy_into(RelocationHolder& holder) const override;
+
+ private:
+  friend class RelocIterator;
+  friend class RelocationHolder;
+  barrier_Relocation() : Relocation(relocInfo::barrier_type) { }
+};
+
+
 class virtual_call_Relocation : public CallRelocation {
 
  public:
diff --git a/src/hotspot/share/gc/shared/barrierSetConfig.hpp b/src/hotspot/share/gc/shared/barrierSetConfig.hpp
index e20e9ef6f7c..76681aa8986 100644
--- a/src/hotspot/share/gc/shared/barrierSetConfig.hpp
+++ b/src/hotspot/share/gc/shared/barrierSetConfig.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -33,6 +33,7 @@
   EPSILONGC_ONLY(f(EpsilonBarrierSet))               \
   G1GC_ONLY(f(G1BarrierSet))                         \
   SHENANDOAHGC_ONLY(f(ShenandoahBarrierSet))         \
+  ZGC_ONLY(f(XBarrierSet))                           \
   ZGC_ONLY(f(ZBarrierSet))
 
 #define FOR_EACH_ABSTRACT_BARRIER_SET_DO(f)          \
diff --git a/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp b/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp
index 27f36f1fee4..9523428821b 100644
--- a/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp
+++ b/src/hotspot/share/gc/shared/barrierSetConfig.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -40,6 +40,7 @@
 #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp"
 #endif
 #if INCLUDE_ZGC
+#include "gc/x/xBarrierSet.inline.hpp"
 #include "gc/z/zBarrierSet.inline.hpp"
 #endif
 
diff --git a/src/hotspot/share/gc/shared/collectedHeap.hpp b/src/hotspot/share/gc/shared/collectedHeap.hpp
index e427e6de668..073db543495 100644
--- a/src/hotspot/share/gc/shared/collectedHeap.hpp
+++ b/src/hotspot/share/gc/shared/collectedHeap.hpp
@@ -95,6 +95,7 @@ class CollectedHeap : public CHeapObj<mtGC> {
   friend class VMStructs;
   friend class JVMCIVMStructs;
   friend class IsGCActiveMark; // Block structured external access to _is_gc_active
+  friend class DisableIsGCActiveMark; // Disable current IsGCActiveMark
   friend class MemAllocator;
   friend class ParallelObjectIterator;
 
diff --git a/src/hotspot/share/gc/shared/gcConfig.cpp b/src/hotspot/share/gc/shared/gcConfig.cpp
index 8eb265b54d9..506b368d6cf 100644
--- a/src/hotspot/share/gc/shared/gcConfig.cpp
+++ b/src/hotspot/share/gc/shared/gcConfig.cpp
@@ -44,7 +44,7 @@
 #include "gc/shenandoah/shenandoahArguments.hpp"
 #endif
 #if INCLUDE_ZGC
-#include "gc/z/zArguments.hpp"
+#include "gc/z/shared/zSharedArguments.hpp"
 #endif
 
 struct IncludedGC {
@@ -62,7 +62,7 @@ struct IncludedGC {
   PARALLELGC_ONLY(static ParallelArguments   parallelArguments;)
     SERIALGC_ONLY(static SerialArguments     serialArguments;)
 SHENANDOAHGC_ONLY(static ShenandoahArguments shenandoahArguments;)
-         ZGC_ONLY(static ZArguments          zArguments;)
+         ZGC_ONLY(static ZSharedArguments    zArguments;)
 
 // Table of included GCs, for translating between command
 // line flag, CollectedHeap::Name and GCArguments instance.
diff --git a/src/hotspot/share/gc/shared/gcConfiguration.cpp b/src/hotspot/share/gc/shared/gcConfiguration.cpp
index 3b6e87fdde0..2e8d3eb2a51 100644
--- a/src/hotspot/share/gc/shared/gcConfiguration.cpp
+++ b/src/hotspot/share/gc/shared/gcConfiguration.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,7 @@
  */
 #include "precompiled.hpp"
 #include "gc/shared/collectedHeap.hpp"
+#include "gc/shared/gc_globals.hpp"
 #include "gc/shared/gcArguments.hpp"
 #include "gc/shared/gcConfiguration.hpp"
 #include "gc/shared/tlab_globals.hpp"
@@ -41,7 +42,15 @@ GCName GCConfiguration::young_collector() const {
     return ParallelScavenge;
   }
 
-  if (UseZGC || UseShenandoahGC) {
+  if (UseZGC) {
+    if (ZGenerational) {
+      return ZMinor;
+    } else {
+      return NA;
+    }
+  }
+
+  if (UseShenandoahGC) {
     return NA;
   }
 
@@ -58,7 +67,11 @@ GCName GCConfiguration::old_collector() const {
   }
 
   if (UseZGC) {
-    return Z;
+    if (ZGenerational) {
+      return ZMajor;
+    } else {
+      return Z;
+    }
   }
 
   if (UseShenandoahGC) {
diff --git a/src/hotspot/share/gc/shared/gcId.cpp b/src/hotspot/share/gc/shared/gcId.cpp
index 97f13a1304d..646de833910 100644
--- a/src/hotspot/share/gc/shared/gcId.cpp
+++ b/src/hotspot/share/gc/shared/gcId.cpp
@@ -30,6 +30,18 @@
 #include "runtime/safepoint.hpp"
 
 uint GCId::_next_id = 0;
+GCIdPrinter GCId::_default_printer;
+GCIdPrinter* GCId::_printer = &_default_printer;
+
+size_t GCIdPrinter::print_gc_id(uint gc_id, char* buf, size_t len) {
+  int ret = jio_snprintf(buf, len, "GC(%u) ", gc_id);
+  assert(ret > 0, "Failed to print prefix. Log buffer too small?");
+  return (size_t)ret;
+}
+
+void GCId::set_printer(GCIdPrinter* printer) {
+  _printer = printer;
+}
 
 NamedThread* currentNamedthread() {
   assert(Thread::current()->is_Named_thread(), "This thread must be NamedThread");
@@ -59,24 +71,20 @@ size_t GCId::print_prefix(char* buf, size_t len) {
   if (thread != nullptr) {
     uint gc_id = current_or_undefined();
     if (gc_id != undefined()) {
-      int ret = jio_snprintf(buf, len, "GC(%u) ", gc_id);
-      assert(ret > 0, "Failed to print prefix. Log buffer too small?");
-      return (size_t)ret;
+      return _printer->print_gc_id(gc_id, buf, len);
     }
   }
   return 0;
 }
 
-GCIdMark::GCIdMark() {
-  assert(currentNamedthread()->gc_id() == GCId::undefined(), "nested");
+GCIdMark::GCIdMark() : _previous_gc_id(currentNamedthread()->gc_id()) {
   currentNamedthread()->set_gc_id(GCId::create());
 }
 
-GCIdMark::GCIdMark(uint gc_id) {
-  assert(currentNamedthread()->gc_id() == GCId::undefined(), "nested");
+GCIdMark::GCIdMark(uint gc_id) : _previous_gc_id(currentNamedthread()->gc_id()) {
   currentNamedthread()->set_gc_id(gc_id);
 }
 
 GCIdMark::~GCIdMark() {
-  currentNamedthread()->set_gc_id(GCId::undefined());
+  currentNamedthread()->set_gc_id(_previous_gc_id);
 }
diff --git a/src/hotspot/share/gc/shared/gcId.hpp b/src/hotspot/share/gc/shared/gcId.hpp
index 1e25d9dc446..d36929b03dc 100644
--- a/src/hotspot/share/gc/shared/gcId.hpp
+++ b/src/hotspot/share/gc/shared/gcId.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,12 @@
 
 #include "memory/allocation.hpp"
 
+class GCIdPrinter : public CHeapObj<mtGC> {
+public:
+  virtual ~GCIdPrinter() {}
+  virtual size_t print_gc_id(uint gc_id, char* buf, size_t len);
+};
+
 class GCId : public AllStatic {
 private:
   friend class GCIdMark;
@@ -35,6 +41,10 @@ private:
   static const uint UNDEFINED = UINT_MAX;
   static uint create();
 
+  // Default printer used unless a custom printer is set
+  static GCIdPrinter _default_printer;
+  static GCIdPrinter* _printer;
+
 public:
   // Returns the currently active GC id. Asserts that there is an active GC id.
   static uint current();
@@ -44,9 +54,14 @@ public:
   static uint peek();
   static uint undefined() { return UNDEFINED; }
   static size_t print_prefix(char* buf, size_t len);
+  // Set a custom GCId printer
+  static void set_printer(GCIdPrinter* printer);
 };
 
 class GCIdMark : public StackObj {
+private:
+  const uint _previous_gc_id;
+
 public:
   GCIdMark();
   GCIdMark(uint gc_id);
diff --git a/src/hotspot/share/gc/shared/gcName.hpp b/src/hotspot/share/gc/shared/gcName.hpp
index ca40c642f3e..3d2dd350ac1 100644
--- a/src/hotspot/share/gc/shared/gcName.hpp
+++ b/src/hotspot/share/gc/shared/gcName.hpp
@@ -35,7 +35,9 @@ enum GCName {
   G1New,
   G1Old,
   G1Full,
-  Z,
+  ZMinor,
+  ZMajor,
+  Z, // Support for the legacy, single-gen mode
   Shenandoah,
   NA,
   GCNameEndSentinel
@@ -52,6 +54,8 @@ class GCNameHelper {
       case G1New: return "G1New";
       case G1Old: return "G1Old";
       case G1Full: return "G1Full";
+      case ZMinor: return "ZGC Minor";
+      case ZMajor: return "ZGC Major";
       case Z: return "Z";
       case Shenandoah: return "Shenandoah";
       case NA: return "N/A";
diff --git a/src/hotspot/share/gc/shared/gcThreadLocalData.hpp b/src/hotspot/share/gc/shared/gcThreadLocalData.hpp
index 404e1520b6b..ba0e6d8fb1a 100644
--- a/src/hotspot/share/gc/shared/gcThreadLocalData.hpp
+++ b/src/hotspot/share/gc/shared/gcThreadLocalData.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -40,6 +40,6 @@
 // should consider placing frequently accessed fields first in
 // T, so that field offsets relative to Thread are small, which
 // often allows for a more compact instruction encoding.
-typedef uint64_t GCThreadLocalData[19]; // 152 bytes
+typedef uint64_t GCThreadLocalData[43]; // 344 bytes
 
 #endif // SHARE_GC_SHARED_GCTHREADLOCALDATA_HPP
diff --git a/src/hotspot/share/gc/shared/gcTraceSend.cpp b/src/hotspot/share/gc/shared/gcTraceSend.cpp
index d7117ffbb40..31ec2871cd0 100644
--- a/src/hotspot/share/gc/shared/gcTraceSend.cpp
+++ b/src/hotspot/share/gc/shared/gcTraceSend.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2012, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -317,11 +317,12 @@ class PhaseSender : public PhaseVisitor {
   }
 
   void visit_concurrent(GCPhase* phase) {
-    assert(phase->level() < 2, "There is only two levels for ConcurrentPhase");
+    assert(phase->level() < 3, "There are only three levels for ConcurrentPhase");
 
     switch (phase->level()) {
       case 0: send_phase<EventGCPhaseConcurrent>(phase); break;
       case 1: send_phase<EventGCPhaseConcurrentLevel1>(phase); break;
+      case 2: send_phase<EventGCPhaseConcurrentLevel2>(phase); break;
       default: /* Ignore sending this phase */ break;
     }
   }
diff --git a/src/hotspot/share/gc/shared/gcVMOperations.cpp b/src/hotspot/share/gc/shared/gcVMOperations.cpp
index e0f988af4e8..ceb4d2cc322 100644
--- a/src/hotspot/share/gc/shared/gcVMOperations.cpp
+++ b/src/hotspot/share/gc/shared/gcVMOperations.cpp
@@ -135,6 +135,17 @@ void VM_GC_Operation::doit_epilogue() {
   VM_GC_Sync_Operation::doit_epilogue();
 }
 
+bool VM_GC_HeapInspection::doit_prologue() {
+  if (_full_gc && UseZGC) {
+    // ZGC cannot perform a synchronous GC cycle from within the VM thread.
+    // So VM_GC_HeapInspection::collect() is a noop. To respect the _full_gc
+    // flag a synchronous GC cycle is performed from the caller thread in the
+    // prologue.
+    Universe::heap()->collect(GCCause::_heap_inspection);
+  }
+  return VM_GC_Operation::doit_prologue();
+}
+
 bool VM_GC_HeapInspection::skip_operation() const {
   return false;
 }
diff --git a/src/hotspot/share/gc/shared/gcVMOperations.hpp b/src/hotspot/share/gc/shared/gcVMOperations.hpp
index 58aa76cb4ba..378ba290367 100644
--- a/src/hotspot/share/gc/shared/gcVMOperations.hpp
+++ b/src/hotspot/share/gc/shared/gcVMOperations.hpp
@@ -171,6 +171,7 @@ class VM_GC_HeapInspection: public VM_GC_Operation {
   ~VM_GC_HeapInspection() {}
   virtual VMOp_Type type() const { return VMOp_GC_HeapInspection; }
   virtual bool skip_operation() const;
+  virtual bool doit_prologue();
   virtual void doit();
  protected:
   bool collect();
diff --git a/src/hotspot/share/gc/shared/gc_globals.hpp b/src/hotspot/share/gc/shared/gc_globals.hpp
index 1bf74a81706..5d720268540 100644
--- a/src/hotspot/share/gc/shared/gc_globals.hpp
+++ b/src/hotspot/share/gc/shared/gc_globals.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -43,7 +43,7 @@
 #include "gc/shenandoah/shenandoah_globals.hpp"
 #endif
 #if INCLUDE_ZGC
-#include "gc/z/z_globals.hpp"
+#include "gc/z/shared/z_shared_globals.hpp"
 #endif
 
 #define GC_FLAGS(develop,                                                   \
@@ -99,7 +99,7 @@
     range,                                                                  \
     constraint))                                                            \
                                                                             \
-  ZGC_ONLY(GC_Z_FLAGS(                                                      \
+  ZGC_ONLY(GC_Z_SHARED_FLAGS(                                               \
     develop,                                                                \
     develop_pd,                                                             \
     product,                                                                \
@@ -125,6 +125,9 @@
   product(bool, UseZGC, false,                                              \
           "Use the Z garbage collector")                                    \
                                                                             \
+  product(bool, ZGenerational, false,                                       \
+          "Use the generational version of ZGC")                            \
+                                                                            \
   product(bool, UseShenandoahGC, false,                                     \
           "Use the Shenandoah garbage collector")                           \
                                                                             \
diff --git a/src/hotspot/share/gc/shared/isGCActiveMark.cpp b/src/hotspot/share/gc/shared/isGCActiveMark.cpp
index 5d5627b1641..c797ff43e5a 100644
--- a/src/hotspot/share/gc/shared/isGCActiveMark.cpp
+++ b/src/hotspot/share/gc/shared/isGCActiveMark.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -42,3 +42,15 @@ IsGCActiveMark::~IsGCActiveMark() {
   assert(heap->is_gc_active(), "Sanity");
   heap->_is_gc_active = false;
 }
+
+DisableIsGCActiveMark::DisableIsGCActiveMark() {
+  CollectedHeap* heap = Universe::heap();
+  assert(heap->is_gc_active(), "Not reentrant");
+  heap->_is_gc_active = false;
+}
+
+DisableIsGCActiveMark::~DisableIsGCActiveMark() {
+  CollectedHeap* heap = Universe::heap();
+  assert(!heap->is_gc_active(), "Sanity");
+  heap->_is_gc_active = true;
+}
diff --git a/src/hotspot/share/gc/shared/isGCActiveMark.hpp b/src/hotspot/share/gc/shared/isGCActiveMark.hpp
index 8b8ce7b4aee..c6d95d8a634 100644
--- a/src/hotspot/share/gc/shared/isGCActiveMark.hpp
+++ b/src/hotspot/share/gc/shared/isGCActiveMark.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -36,4 +36,10 @@ class IsGCActiveMark : public StackObj {
   ~IsGCActiveMark();
 };
 
+class DisableIsGCActiveMark : public StackObj {
+ public:
+  DisableIsGCActiveMark();
+  ~DisableIsGCActiveMark();
+};
+
 #endif // SHARE_GC_SHARED_ISGCACTIVEMARK_HPP
diff --git a/src/hotspot/share/gc/shared/suspendibleThreadSet.cpp b/src/hotspot/share/gc/shared/suspendibleThreadSet.cpp
index 163da887bb5..a0d83ac298d 100644
--- a/src/hotspot/share/gc/shared/suspendibleThreadSet.cpp
+++ b/src/hotspot/share/gc/shared/suspendibleThreadSet.cpp
@@ -91,7 +91,6 @@ void SuspendibleThreadSet::yield_slow() {
 }
 
 void SuspendibleThreadSet::synchronize() {
-  assert(Thread::current()->is_VM_thread(), "Must be the VM thread");
   if (ConcGCYieldTimeout > 0) {
     _suspend_all_start = os::elapsedTime();
   }
@@ -126,7 +125,6 @@ void SuspendibleThreadSet::synchronize() {
 }
 
 void SuspendibleThreadSet::desynchronize() {
-  assert(Thread::current()->is_VM_thread(), "Must be the VM thread");
   MonitorLocker ml(STS_lock, Mutex::_no_safepoint_check_flag);
   assert(should_yield(), "STS not synchronizing");
   assert(is_synchronized(), "STS not synchronized");
diff --git a/src/hotspot/share/gc/shared/vmStructs_gc.hpp b/src/hotspot/share/gc/shared/vmStructs_gc.hpp
index 2db0f466d34..acd153cd45d 100644
--- a/src/hotspot/share/gc/shared/vmStructs_gc.hpp
+++ b/src/hotspot/share/gc/shared/vmStructs_gc.hpp
@@ -50,7 +50,7 @@
 #include "gc/shenandoah/vmStructs_shenandoah.hpp"
 #endif
 #if INCLUDE_ZGC
-#include "gc/z/vmStructs_z.hpp"
+#include "gc/z/shared/vmStructs_z_shared.hpp"
 #endif
 
 #define VM_STRUCTS_GC(nonstatic_field,                                                                                               \
@@ -72,9 +72,9 @@
   SHENANDOAHGC_ONLY(VM_STRUCTS_SHENANDOAH(nonstatic_field,                                                                           \
                                volatile_nonstatic_field,                                                                             \
                                static_field))                                                                                        \
-  ZGC_ONLY(VM_STRUCTS_ZGC(nonstatic_field,                                                                                           \
-                          volatile_nonstatic_field,                                                                                  \
-                          static_field))                                                                                             \
+  ZGC_ONLY(VM_STRUCTS_Z_SHARED(nonstatic_field,                                                                                      \
+                               volatile_nonstatic_field,                                                                             \
+                               static_field))                                                                                        \
                                                                                                                                      \
   /**********************************************************************************/                                               \
   /* Generation and Space hierarchies                                               */                                               \
@@ -146,9 +146,9 @@
   SHENANDOAHGC_ONLY(VM_TYPES_SHENANDOAH(declare_type,                     \
                              declare_toplevel_type,                       \
                              declare_integer_type))                       \
-  ZGC_ONLY(VM_TYPES_ZGC(declare_type,                                     \
-                        declare_toplevel_type,                            \
-                        declare_integer_type))                            \
+  ZGC_ONLY(VM_TYPES_Z_SHARED(declare_type,                                \
+                             declare_toplevel_type,                       \
+                             declare_integer_type))                       \
                                                                           \
   /******************************************/                            \
   /* Generation and space hierarchies       */                            \
@@ -210,8 +210,8 @@
                                           declare_constant_with_value))     \
   SHENANDOAHGC_ONLY(VM_INT_CONSTANTS_SHENANDOAH(declare_constant,           \
                                      declare_constant_with_value))          \
-  ZGC_ONLY(VM_INT_CONSTANTS_ZGC(declare_constant,                           \
-                                declare_constant_with_value))               \
+  ZGC_ONLY(VM_INT_CONSTANTS_Z_SHARED(declare_constant,                      \
+                                     declare_constant_with_value))          \
                                                                             \
   /********************************************/                            \
   /* Generation and Space Hierarchy Constants */                            \
@@ -243,6 +243,6 @@
   declare_constant(Generation::GenGrain)                                    \
 
 #define VM_LONG_CONSTANTS_GC(declare_constant)                              \
-  ZGC_ONLY(VM_LONG_CONSTANTS_ZGC(declare_constant))
+  ZGC_ONLY(VM_LONG_CONSTANTS_Z_SHARED(declare_constant))
 
 #endif // SHARE_GC_SHARED_VMSTRUCTS_GC_HPP
diff --git a/src/hotspot/share/gc/shared/workerThread.cpp b/src/hotspot/share/gc/shared/workerThread.cpp
index 14f9c349d63..9fc9f66c55b 100644
--- a/src/hotspot/share/gc/shared/workerThread.cpp
+++ b/src/hotspot/share/gc/shared/workerThread.cpp
@@ -141,8 +141,40 @@ void WorkerThreads::threads_do(ThreadClosure* tc) const {
   }
 }
 
+void WorkerThreads::set_indirectly_suspendible_threads() {
+#ifdef ASSERT
+  class SetIndirectlySuspendibleThreadClosure : public ThreadClosure {
+    virtual void do_thread(Thread* thread) {
+      thread->set_indirectly_suspendible_thread();
+    }
+  };
+
+  if (Thread::current()->is_suspendible_thread()) {
+    SetIndirectlySuspendibleThreadClosure cl;
+    threads_do(&cl);
+  }
+#endif
+}
+
+void WorkerThreads::clear_indirectly_suspendible_threads() {
+#ifdef ASSERT
+  class ClearIndirectlySuspendibleThreadClosure : public ThreadClosure {
+    virtual void do_thread(Thread* thread) {
+      thread->clear_indirectly_suspendible_thread();
+    }
+  };
+
+  if (Thread::current()->is_suspendible_thread()) {
+    ClearIndirectlySuspendibleThreadClosure cl;
+    threads_do(&cl);
+  }
+#endif
+}
+
 void WorkerThreads::run_task(WorkerTask* task) {
+  set_indirectly_suspendible_threads();
   _dispatcher.coordinator_distribute_task(task, _active_workers);
+  clear_indirectly_suspendible_threads();
 }
 
 void WorkerThreads::run_task(WorkerTask* task, uint num_workers) {
diff --git a/src/hotspot/share/gc/shared/workerThread.hpp b/src/hotspot/share/gc/shared/workerThread.hpp
index bdb61f34ed9..d3b246c0930 100644
--- a/src/hotspot/share/gc/shared/workerThread.hpp
+++ b/src/hotspot/share/gc/shared/workerThread.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -93,6 +93,9 @@ private:
 
   WorkerThread* create_worker(uint name_suffix);
 
+  void set_indirectly_suspendible_threads();
+  void clear_indirectly_suspendible_threads();
+
 protected:
   virtual void on_create_worker(WorkerThread* worker) {}
 
diff --git a/src/hotspot/share/gc/x/c1/xBarrierSetC1.cpp b/src/hotspot/share/gc/x/c1/xBarrierSetC1.cpp
new file mode 100644
index 00000000000..ddc997900df
--- /dev/null
+++ b/src/hotspot/share/gc/x/c1/xBarrierSetC1.cpp
@@ -0,0 +1,237 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "c1/c1_LIR.hpp"
+#include "c1/c1_LIRGenerator.hpp"
+#include "c1/c1_CodeStubs.hpp"
+#include "gc/x/c1/xBarrierSetC1.hpp"
+#include "gc/x/xBarrierSet.hpp"
+#include "gc/x/xBarrierSetAssembler.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+#include "utilities/macros.hpp"
+
+XLoadBarrierStubC1::XLoadBarrierStubC1(LIRAccess& access, LIR_Opr ref, address runtime_stub) :
+    _decorators(access.decorators()),
+    _ref_addr(access.resolved_addr()),
+    _ref(ref),
+    _tmp(LIR_OprFact::illegalOpr),
+    _runtime_stub(runtime_stub) {
+
+  assert(_ref_addr->is_address(), "Must be an address");
+  assert(_ref->is_register(), "Must be a register");
+
+  // Allocate tmp register if needed
+  if (_ref_addr->as_address_ptr()->index()->is_valid() ||
+      _ref_addr->as_address_ptr()->disp() != 0) {
+    // Has index or displacement, need tmp register to load address into
+    _tmp = access.gen()->new_pointer_register();
+  }
+
+  FrameMap* f = Compilation::current()->frame_map();
+  f->update_reserved_argument_area_size(2 * BytesPerWord);
+}
+
+DecoratorSet XLoadBarrierStubC1::decorators() const {
+  return _decorators;
+}
+
+LIR_Opr XLoadBarrierStubC1::ref() const {
+  return _ref;
+}
+
+LIR_Opr XLoadBarrierStubC1::ref_addr() const {
+  return _ref_addr;
+}
+
+LIR_Opr XLoadBarrierStubC1::tmp() const {
+  return _tmp;
+}
+
+address XLoadBarrierStubC1::runtime_stub() const {
+  return _runtime_stub;
+}
+
+void XLoadBarrierStubC1::visit(LIR_OpVisitState* visitor) {
+  visitor->do_slow_case();
+  visitor->do_input(_ref_addr);
+  visitor->do_output(_ref);
+  if (_tmp->is_valid()) {
+    visitor->do_temp(_tmp);
+  }
+}
+
+void XLoadBarrierStubC1::emit_code(LIR_Assembler* ce) {
+  XBarrierSet::assembler()->generate_c1_load_barrier_stub(ce, this);
+}
+
+#ifndef PRODUCT
+void XLoadBarrierStubC1::print_name(outputStream* out) const {
+  out->print("XLoadBarrierStubC1");
+}
+#endif // PRODUCT
+
+class LIR_OpXLoadBarrierTest : public LIR_Op {
+private:
+  LIR_Opr _opr;
+
+public:
+  LIR_OpXLoadBarrierTest(LIR_Opr opr) :
+      LIR_Op(lir_xloadbarrier_test, LIR_OprFact::illegalOpr, NULL),
+      _opr(opr) {}
+
+  virtual void visit(LIR_OpVisitState* state) {
+    state->do_input(_opr);
+  }
+
+  virtual void emit_code(LIR_Assembler* ce) {
+    XBarrierSet::assembler()->generate_c1_load_barrier_test(ce, _opr);
+  }
+
+  virtual void print_instr(outputStream* out) const {
+    _opr->print(out);
+    out->print(" ");
+  }
+
+#ifndef PRODUCT
+  virtual const char* name() const {
+    return "lir_z_load_barrier_test";
+  }
+#endif // PRODUCT
+};
+
+static bool barrier_needed(LIRAccess& access) {
+  return XBarrierSet::barrier_needed(access.decorators(), access.type());
+}
+
+XBarrierSetC1::XBarrierSetC1() :
+    _load_barrier_on_oop_field_preloaded_runtime_stub(NULL),
+    _load_barrier_on_weak_oop_field_preloaded_runtime_stub(NULL) {}
+
+address XBarrierSetC1::load_barrier_on_oop_field_preloaded_runtime_stub(DecoratorSet decorators) const {
+  assert((decorators & ON_PHANTOM_OOP_REF) == 0, "Unsupported decorator");
+  //assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Unsupported decorator");
+
+  if ((decorators & ON_WEAK_OOP_REF) != 0) {
+    return _load_barrier_on_weak_oop_field_preloaded_runtime_stub;
+  } else {
+    return _load_barrier_on_oop_field_preloaded_runtime_stub;
+  }
+}
+
+#ifdef ASSERT
+#define __ access.gen()->lir(__FILE__, __LINE__)->
+#else
+#define __ access.gen()->lir()->
+#endif
+
+void XBarrierSetC1::load_barrier(LIRAccess& access, LIR_Opr result) const {
+  // Fast path
+  __ append(new LIR_OpXLoadBarrierTest(result));
+
+  // Slow path
+  const address runtime_stub = load_barrier_on_oop_field_preloaded_runtime_stub(access.decorators());
+  CodeStub* const stub = new XLoadBarrierStubC1(access, result, runtime_stub);
+  __ branch(lir_cond_notEqual, stub);
+  __ branch_destination(stub->continuation());
+}
+
+LIR_Opr XBarrierSetC1::resolve_address(LIRAccess& access, bool resolve_in_register) {
+  // We must resolve in register when patching. This is to avoid
+  // having a patch area in the load barrier stub, since the call
+  // into the runtime to patch will not have the proper oop map.
+  const bool patch_before_barrier = barrier_needed(access) && (access.decorators() & C1_NEEDS_PATCHING) != 0;
+  return BarrierSetC1::resolve_address(access, resolve_in_register || patch_before_barrier);
+}
+
+#undef __
+
+void XBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) {
+  BarrierSetC1::load_at_resolved(access, result);
+
+  if (barrier_needed(access)) {
+    load_barrier(access, result);
+  }
+}
+
+static void pre_load_barrier(LIRAccess& access) {
+  DecoratorSet decorators = access.decorators();
+
+  // Downgrade access to MO_UNORDERED
+  decorators = (decorators & ~MO_DECORATOR_MASK) | MO_UNORDERED;
+
+  // Remove ACCESS_WRITE
+  decorators = (decorators & ~ACCESS_WRITE);
+
+  // Generate synthetic load at
+  access.gen()->access_load_at(decorators,
+                               access.type(),
+                               access.base().item(),
+                               access.offset().opr(),
+                               access.gen()->new_register(access.type()),
+                               NULL /* patch_emit_info */,
+                               NULL /* load_emit_info */);
+}
+
+LIR_Opr XBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) {
+  if (barrier_needed(access)) {
+    pre_load_barrier(access);
+  }
+
+  return BarrierSetC1::atomic_xchg_at_resolved(access, value);
+}
+
+LIR_Opr XBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) {
+  if (barrier_needed(access)) {
+    pre_load_barrier(access);
+  }
+
+  return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
+}
+
+class XLoadBarrierRuntimeStubCodeGenClosure : public StubAssemblerCodeGenClosure {
+private:
+  const DecoratorSet _decorators;
+
+public:
+  XLoadBarrierRuntimeStubCodeGenClosure(DecoratorSet decorators) :
+      _decorators(decorators) {}
+
+  virtual OopMapSet* generate_code(StubAssembler* sasm) {
+    XBarrierSet::assembler()->generate_c1_load_barrier_runtime_stub(sasm, _decorators);
+    return NULL;
+  }
+};
+
+static address generate_c1_runtime_stub(BufferBlob* blob, DecoratorSet decorators, const char* name) {
+  XLoadBarrierRuntimeStubCodeGenClosure cl(decorators);
+  CodeBlob* const code_blob = Runtime1::generate_blob(blob, -1 /* stub_id */, name, false /* expect_oop_map*/, &cl);
+  return code_blob->code_begin();
+}
+
+void XBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* blob) {
+  _load_barrier_on_oop_field_preloaded_runtime_stub =
+    generate_c1_runtime_stub(blob, ON_STRONG_OOP_REF, "load_barrier_on_oop_field_preloaded_runtime_stub");
+  _load_barrier_on_weak_oop_field_preloaded_runtime_stub =
+    generate_c1_runtime_stub(blob, ON_WEAK_OOP_REF, "load_barrier_on_weak_oop_field_preloaded_runtime_stub");
+}
diff --git a/src/hotspot/share/gc/x/c1/xBarrierSetC1.hpp b/src/hotspot/share/gc/x/c1/xBarrierSetC1.hpp
new file mode 100644
index 00000000000..26c2e142cdf
--- /dev/null
+++ b/src/hotspot/share/gc/x/c1/xBarrierSetC1.hpp
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_C1_XBARRIERSETC1_HPP
+#define SHARE_GC_X_C1_XBARRIERSETC1_HPP
+
+#include "c1/c1_CodeStubs.hpp"
+#include "c1/c1_IR.hpp"
+#include "c1/c1_LIR.hpp"
+#include "gc/shared/c1/barrierSetC1.hpp"
+#include "oops/accessDecorators.hpp"
+
+class XLoadBarrierStubC1 : public CodeStub {
+private:
+  DecoratorSet _decorators;
+  LIR_Opr      _ref_addr;
+  LIR_Opr      _ref;
+  LIR_Opr      _tmp;
+  address      _runtime_stub;
+
+public:
+  XLoadBarrierStubC1(LIRAccess& access, LIR_Opr ref, address runtime_stub);
+
+  DecoratorSet decorators() const;
+  LIR_Opr ref() const;
+  LIR_Opr ref_addr() const;
+  LIR_Opr tmp() const;
+  address runtime_stub() const;
+
+  virtual void emit_code(LIR_Assembler* ce);
+  virtual void visit(LIR_OpVisitState* visitor);
+
+#ifndef PRODUCT
+  virtual void print_name(outputStream* out) const;
+#endif // PRODUCT
+};
+
+class XBarrierSetC1 : public BarrierSetC1 {
+private:
+  address _load_barrier_on_oop_field_preloaded_runtime_stub;
+  address _load_barrier_on_weak_oop_field_preloaded_runtime_stub;
+
+  address load_barrier_on_oop_field_preloaded_runtime_stub(DecoratorSet decorators) const;
+  void load_barrier(LIRAccess& access, LIR_Opr result) const;
+
+protected:
+  virtual LIR_Opr resolve_address(LIRAccess& access, bool resolve_in_register);
+  virtual void load_at_resolved(LIRAccess& access, LIR_Opr result);
+  virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value);
+  virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value);
+
+public:
+  XBarrierSetC1();
+
+  virtual void generate_c1_runtime_stubs(BufferBlob* blob);
+};
+
+#endif // SHARE_GC_X_C1_XBARRIERSETC1_HPP
diff --git a/src/hotspot/share/gc/x/c2/xBarrierSetC2.cpp b/src/hotspot/share/gc/x/c2/xBarrierSetC2.cpp
new file mode 100644
index 00000000000..5ec0558cc78
--- /dev/null
+++ b/src/hotspot/share/gc/x/c2/xBarrierSetC2.cpp
@@ -0,0 +1,583 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "classfile/javaClasses.hpp"
+#include "gc/x/c2/xBarrierSetC2.hpp"
+#include "gc/x/xBarrierSet.hpp"
+#include "gc/x/xBarrierSetAssembler.hpp"
+#include "gc/x/xBarrierSetRuntime.hpp"
+#include "opto/arraycopynode.hpp"
+#include "opto/addnode.hpp"
+#include "opto/block.hpp"
+#include "opto/compile.hpp"
+#include "opto/graphKit.hpp"
+#include "opto/machnode.hpp"
+#include "opto/macro.hpp"
+#include "opto/memnode.hpp"
+#include "opto/node.hpp"
+#include "opto/output.hpp"
+#include "opto/regalloc.hpp"
+#include "opto/rootnode.hpp"
+#include "opto/runtime.hpp"
+#include "opto/type.hpp"
+#include "utilities/growableArray.hpp"
+#include "utilities/macros.hpp"
+
+class XBarrierSetC2State : public ArenaObj {
+private:
+  GrowableArray<XLoadBarrierStubC2*>* _stubs;
+  Node_Array                          _live;
+
+public:
+  XBarrierSetC2State(Arena* arena) :
+    _stubs(new (arena) GrowableArray<XLoadBarrierStubC2*>(arena, 8,  0, NULL)),
+    _live(arena) {}
+
+  GrowableArray<XLoadBarrierStubC2*>* stubs() {
+    return _stubs;
+  }
+
+  RegMask* live(const Node* node) {
+    if (!node->is_Mach()) {
+      // Don't need liveness for non-MachNodes
+      return NULL;
+    }
+
+    const MachNode* const mach = node->as_Mach();
+    if (mach->barrier_data() == XLoadBarrierElided) {
+      // Don't need liveness data for nodes without barriers
+      return NULL;
+    }
+
+    RegMask* live = (RegMask*)_live[node->_idx];
+    if (live == NULL) {
+      live = new (Compile::current()->comp_arena()->AmallocWords(sizeof(RegMask))) RegMask();
+      _live.map(node->_idx, (Node*)live);
+    }
+
+    return live;
+  }
+};
+
+static XBarrierSetC2State* barrier_set_state() {
+  return reinterpret_cast<XBarrierSetC2State*>(Compile::current()->barrier_set_state());
+}
+
+XLoadBarrierStubC2* XLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data) {
+  XLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) XLoadBarrierStubC2(node, ref_addr, ref, tmp, barrier_data);
+  if (!Compile::current()->output()->in_scratch_emit_size()) {
+    barrier_set_state()->stubs()->append(stub);
+  }
+
+  return stub;
+}
+
+XLoadBarrierStubC2::XLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data) :
+    _node(node),
+    _ref_addr(ref_addr),
+    _ref(ref),
+    _tmp(tmp),
+    _barrier_data(barrier_data),
+    _entry(),
+    _continuation() {
+  assert_different_registers(ref, ref_addr.base());
+  assert_different_registers(ref, ref_addr.index());
+}
+
+Address XLoadBarrierStubC2::ref_addr() const {
+  return _ref_addr;
+}
+
+Register XLoadBarrierStubC2::ref() const {
+  return _ref;
+}
+
+Register XLoadBarrierStubC2::tmp() const {
+  return _tmp;
+}
+
+address XLoadBarrierStubC2::slow_path() const {
+  DecoratorSet decorators = DECORATORS_NONE;
+  if (_barrier_data & XLoadBarrierStrong) {
+    decorators |= ON_STRONG_OOP_REF;
+  }
+  if (_barrier_data & XLoadBarrierWeak) {
+    decorators |= ON_WEAK_OOP_REF;
+  }
+  if (_barrier_data & XLoadBarrierPhantom) {
+    decorators |= ON_PHANTOM_OOP_REF;
+  }
+  if (_barrier_data & XLoadBarrierNoKeepalive) {
+    decorators |= AS_NO_KEEPALIVE;
+  }
+  return XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators);
+}
+
+RegMask& XLoadBarrierStubC2::live() const {
+  RegMask* mask = barrier_set_state()->live(_node);
+  assert(mask != NULL, "must be mach-node with barrier");
+  return *mask;
+}
+
+Label* XLoadBarrierStubC2::entry() {
+  // The _entry will never be bound when in_scratch_emit_size() is true.
+  // However, we still need to return a label that is not bound now, but
+  // will eventually be bound. Any label will do, as it will only act as
+  // a placeholder, so we return the _continuation label.
+  return Compile::current()->output()->in_scratch_emit_size() ? &_continuation : &_entry;
+}
+
+Label* XLoadBarrierStubC2::continuation() {
+  return &_continuation;
+}
+
+void* XBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
+  return new (comp_arena) XBarrierSetC2State(comp_arena);
+}
+
+void XBarrierSetC2::late_barrier_analysis() const {
+  analyze_dominating_barriers();
+  compute_liveness_at_stubs();
+}
+
+void XBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
+  MacroAssembler masm(&cb);
+  GrowableArray<XLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
+
+  for (int i = 0; i < stubs->length(); i++) {
+    // Make sure there is enough space in the code buffer
+    if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == NULL) {
+      ciEnv::current()->record_failure("CodeCache is full");
+      return;
+    }
+
+    XBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
+  }
+
+  masm.flush();
+}
+
+int XBarrierSetC2::estimate_stub_size() const {
+  Compile* const C = Compile::current();
+  BufferBlob* const blob = C->output()->scratch_buffer_blob();
+  GrowableArray<XLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
+  int size = 0;
+
+  for (int i = 0; i < stubs->length(); i++) {
+    CodeBuffer cb(blob->content_begin(), (address)C->output()->scratch_locs_memory() - blob->content_begin());
+    MacroAssembler masm(&cb);
+    XBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
+    size += cb.insts_size();
+  }
+
+  return size;
+}
+
+static void set_barrier_data(C2Access& access) {
+  if (XBarrierSet::barrier_needed(access.decorators(), access.type())) {
+    uint8_t barrier_data = 0;
+
+    if (access.decorators() & ON_PHANTOM_OOP_REF) {
+      barrier_data |= XLoadBarrierPhantom;
+    } else if (access.decorators() & ON_WEAK_OOP_REF) {
+      barrier_data |= XLoadBarrierWeak;
+    } else {
+      barrier_data |= XLoadBarrierStrong;
+    }
+
+    if (access.decorators() & AS_NO_KEEPALIVE) {
+      barrier_data |= XLoadBarrierNoKeepalive;
+    }
+
+    access.set_barrier_data(barrier_data);
+  }
+}
+
+Node* XBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
+  set_barrier_data(access);
+  return BarrierSetC2::load_at_resolved(access, val_type);
+}
+
+Node* XBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
+                                                    Node* new_val, const Type* val_type) const {
+  set_barrier_data(access);
+  return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, val_type);
+}
+
+Node* XBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access, Node* expected_val,
+                                                     Node* new_val, const Type* value_type) const {
+  set_barrier_data(access);
+  return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type);
+}
+
+Node* XBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& access, Node* new_val, const Type* val_type) const {
+  set_barrier_data(access);
+  return BarrierSetC2::atomic_xchg_at_resolved(access, new_val, val_type);
+}
+
+bool XBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc, BasicType type,
+                                                    bool is_clone, bool is_clone_instance,
+                                                    ArrayCopyPhase phase) const {
+  if (phase == ArrayCopyPhase::Parsing) {
+    return false;
+  }
+  if (phase == ArrayCopyPhase::Optimization) {
+    return is_clone_instance;
+  }
+  // else ArrayCopyPhase::Expansion
+  return type == T_OBJECT || type == T_ARRAY;
+}
+
+// This TypeFunc assumes a 64bit system
+static const TypeFunc* clone_type() {
+  // Create input type (domain)
+  const Type** domain_fields = TypeTuple::fields(4);
+  domain_fields[TypeFunc::Parms + 0] = TypeInstPtr::NOTNULL;  // src
+  domain_fields[TypeFunc::Parms + 1] = TypeInstPtr::NOTNULL;  // dst
+  domain_fields[TypeFunc::Parms + 2] = TypeLong::LONG;        // size lower
+  domain_fields[TypeFunc::Parms + 3] = Type::HALF;            // size upper
+  const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + 4, domain_fields);
+
+  // Create result type (range)
+  const Type** range_fields = TypeTuple::fields(0);
+  const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 0, range_fields);
+
+  return TypeFunc::make(domain, range);
+}
+
+#define XTOP LP64_ONLY(COMMA phase->top())
+
+void XBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
+  Node* const src = ac->in(ArrayCopyNode::Src);
+  const TypeAryPtr* ary_ptr = src->get_ptr_type()->isa_aryptr();
+
+  if (ac->is_clone_array() && ary_ptr != NULL) {
+    BasicType bt = ary_ptr->elem()->array_element_basic_type();
+    if (is_reference_type(bt)) {
+      // Clone object array
+      bt = T_OBJECT;
+    } else {
+      // Clone primitive array
+      bt = T_LONG;
+    }
+
+    Node* ctrl = ac->in(TypeFunc::Control);
+    Node* mem = ac->in(TypeFunc::Memory);
+    Node* src = ac->in(ArrayCopyNode::Src);
+    Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
+    Node* dest = ac->in(ArrayCopyNode::Dest);
+    Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
+    Node* length = ac->in(ArrayCopyNode::Length);
+
+    if (bt == T_OBJECT) {
+      // BarrierSetC2::clone sets the offsets via BarrierSetC2::arraycopy_payload_base_offset
+      // which 8-byte aligns them to allow for word size copies. Make sure the offsets point
+      // to the first element in the array when cloning object arrays. Otherwise, load
+      // barriers are applied to parts of the header. Also adjust the length accordingly.
+      assert(src_offset == dest_offset, "should be equal");
+      jlong offset = src_offset->get_long();
+      if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
+        assert(!UseCompressedClassPointers, "should only happen without compressed class pointers");
+        assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
+        length = phase->transform_later(new SubLNode(length, phase->longcon(1))); // Size is in longs
+        src_offset = phase->longcon(arrayOopDesc::base_offset_in_bytes(T_OBJECT));
+        dest_offset = src_offset;
+      }
+    }
+    Node* payload_src = phase->basic_plus_adr(src, src_offset);
+    Node* payload_dst = phase->basic_plus_adr(dest, dest_offset);
+
+    const char* copyfunc_name = "arraycopy";
+    address     copyfunc_addr = phase->basictype2arraycopy(bt, NULL, NULL, true, copyfunc_name, true);
+
+    const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
+    const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
+
+    Node* call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
+    phase->transform_later(call);
+
+    phase->igvn().replace_node(ac, call);
+    return;
+  }
+
+  // Clone instance
+  Node* const ctrl       = ac->in(TypeFunc::Control);
+  Node* const mem        = ac->in(TypeFunc::Memory);
+  Node* const dst        = ac->in(ArrayCopyNode::Dest);
+  Node* const size       = ac->in(ArrayCopyNode::Length);
+
+  assert(size->bottom_type()->is_long(), "Should be long");
+
+  // The native clone we are calling here expects the instance size in words
+  // Add header/offset size to payload size to get instance size.
+  Node* const base_offset = phase->longcon(arraycopy_payload_base_offset(ac->is_clone_array()) >> LogBytesPerLong);
+  Node* const full_size = phase->transform_later(new AddLNode(size, base_offset));
+
+  Node* const call = phase->make_leaf_call(ctrl,
+                                           mem,
+                                           clone_type(),
+                                           XBarrierSetRuntime::clone_addr(),
+                                           "XBarrierSetRuntime::clone",
+                                           TypeRawPtr::BOTTOM,
+                                           src,
+                                           dst,
+                                           full_size,
+                                           phase->top());
+  phase->transform_later(call);
+  phase->igvn().replace_node(ac, call);
+}
+
+#undef XTOP
+
+// == Dominating barrier elision ==
+
+static bool block_has_safepoint(const Block* block, uint from, uint to) {
+  for (uint i = from; i < to; i++) {
+    if (block->get_node(i)->is_MachSafePoint()) {
+      // Safepoint found
+      return true;
+    }
+  }
+
+  // Safepoint not found
+  return false;
+}
+
+static bool block_has_safepoint(const Block* block) {
+  return block_has_safepoint(block, 0, block->number_of_nodes());
+}
+
+static uint block_index(const Block* block, const Node* node) {
+  for (uint j = 0; j < block->number_of_nodes(); ++j) {
+    if (block->get_node(j) == node) {
+      return j;
+    }
+  }
+  ShouldNotReachHere();
+  return 0;
+}
+
+void XBarrierSetC2::analyze_dominating_barriers() const {
+  ResourceMark rm;
+  Compile* const C = Compile::current();
+  PhaseCFG* const cfg = C->cfg();
+  Block_List worklist;
+  Node_List mem_ops;
+  Node_List barrier_loads;
+
+  // Step 1 - Find accesses, and track them in lists
+  for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
+    const Block* const block = cfg->get_block(i);
+    for (uint j = 0; j < block->number_of_nodes(); ++j) {
+      const Node* const node = block->get_node(j);
+      if (!node->is_Mach()) {
+        continue;
+      }
+
+      MachNode* const mach = node->as_Mach();
+      switch (mach->ideal_Opcode()) {
+      case Op_LoadP:
+        if ((mach->barrier_data() & XLoadBarrierStrong) != 0) {
+          barrier_loads.push(mach);
+        }
+        if ((mach->barrier_data() & (XLoadBarrierStrong | XLoadBarrierNoKeepalive)) ==
+            XLoadBarrierStrong) {
+          mem_ops.push(mach);
+        }
+        break;
+      case Op_CompareAndExchangeP:
+      case Op_CompareAndSwapP:
+      case Op_GetAndSetP:
+        if ((mach->barrier_data() & XLoadBarrierStrong) != 0) {
+          barrier_loads.push(mach);
+        }
+      case Op_StoreP:
+        mem_ops.push(mach);
+        break;
+
+      default:
+        break;
+      }
+    }
+  }
+
+  // Step 2 - Find dominating accesses for each load
+  for (uint i = 0; i < barrier_loads.size(); i++) {
+    MachNode* const load = barrier_loads.at(i)->as_Mach();
+    const TypePtr* load_adr_type = NULL;
+    intptr_t load_offset = 0;
+    const Node* const load_obj = load->get_base_and_disp(load_offset, load_adr_type);
+    Block* const load_block = cfg->get_block_for_node(load);
+    const uint load_index = block_index(load_block, load);
+
+    for (uint j = 0; j < mem_ops.size(); j++) {
+      MachNode* mem = mem_ops.at(j)->as_Mach();
+      const TypePtr* mem_adr_type = NULL;
+      intptr_t mem_offset = 0;
+      const Node* mem_obj = mem->get_base_and_disp(mem_offset, mem_adr_type);
+      Block* mem_block = cfg->get_block_for_node(mem);
+      uint mem_index = block_index(mem_block, mem);
+
+      if (load_obj == NodeSentinel || mem_obj == NodeSentinel ||
+          load_obj == NULL || mem_obj == NULL ||
+          load_offset < 0 || mem_offset < 0) {
+        continue;
+      }
+
+      if (mem_obj != load_obj || mem_offset != load_offset) {
+        // Not the same addresses, not a candidate
+        continue;
+      }
+
+      if (load_block == mem_block) {
+        // Earlier accesses in the same block
+        if (mem_index < load_index && !block_has_safepoint(mem_block, mem_index + 1, load_index)) {
+          load->set_barrier_data(XLoadBarrierElided);
+        }
+      } else if (mem_block->dominates(load_block)) {
+        // Dominating block? Look around for safepoints
+        ResourceMark rm;
+        Block_List stack;
+        VectorSet visited;
+        stack.push(load_block);
+        bool safepoint_found = block_has_safepoint(load_block);
+        while (!safepoint_found && stack.size() > 0) {
+          Block* block = stack.pop();
+          if (visited.test_set(block->_pre_order)) {
+            continue;
+          }
+          if (block_has_safepoint(block)) {
+            safepoint_found = true;
+            break;
+          }
+          if (block == mem_block) {
+            continue;
+          }
+
+          // Push predecessor blocks
+          for (uint p = 1; p < block->num_preds(); ++p) {
+            Block* pred = cfg->get_block_for_node(block->pred(p));
+            stack.push(pred);
+          }
+        }
+
+        if (!safepoint_found) {
+          load->set_barrier_data(XLoadBarrierElided);
+        }
+      }
+    }
+  }
+}
+
+// == Reduced spilling optimization ==
+
+void XBarrierSetC2::compute_liveness_at_stubs() const {
+  ResourceMark rm;
+  Compile* const C = Compile::current();
+  Arena* const A = Thread::current()->resource_area();
+  PhaseCFG* const cfg = C->cfg();
+  PhaseRegAlloc* const regalloc = C->regalloc();
+  RegMask* const live = NEW_ARENA_ARRAY(A, RegMask, cfg->number_of_blocks() * sizeof(RegMask));
+  XBarrierSetAssembler* const bs = XBarrierSet::assembler();
+  Block_List worklist;
+
+  for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
+    new ((void*)(live + i)) RegMask();
+    worklist.push(cfg->get_block(i));
+  }
+
+  while (worklist.size() > 0) {
+    const Block* const block = worklist.pop();
+    RegMask& old_live = live[block->_pre_order];
+    RegMask new_live;
+
+    // Initialize to union of successors
+    for (uint i = 0; i < block->_num_succs; i++) {
+      const uint succ_id = block->_succs[i]->_pre_order;
+      new_live.OR(live[succ_id]);
+    }
+
+    // Walk block backwards, computing liveness
+    for (int i = block->number_of_nodes() - 1; i >= 0; --i) {
+      const Node* const node = block->get_node(i);
+
+      // Remove def bits
+      const OptoReg::Name first = bs->refine_register(node, regalloc->get_reg_first(node));
+      const OptoReg::Name second = bs->refine_register(node, regalloc->get_reg_second(node));
+      if (first != OptoReg::Bad) {
+        new_live.Remove(first);
+      }
+      if (second != OptoReg::Bad) {
+        new_live.Remove(second);
+      }
+
+      // Add use bits
+      for (uint j = 1; j < node->req(); ++j) {
+        const Node* const use = node->in(j);
+        const OptoReg::Name first = bs->refine_register(use, regalloc->get_reg_first(use));
+        const OptoReg::Name second = bs->refine_register(use, regalloc->get_reg_second(use));
+        if (first != OptoReg::Bad) {
+          new_live.Insert(first);
+        }
+        if (second != OptoReg::Bad) {
+          new_live.Insert(second);
+        }
+      }
+
+      // If this node tracks liveness, update it
+      RegMask* const regs = barrier_set_state()->live(node);
+      if (regs != NULL) {
+        regs->OR(new_live);
+      }
+    }
+
+    // Now at block top, see if we have any changes
+    new_live.SUBTRACT(old_live);
+    if (new_live.is_NotEmpty()) {
+      // Liveness has refined, update and propagate to prior blocks
+      old_live.OR(new_live);
+      for (uint i = 1; i < block->num_preds(); ++i) {
+        Block* const pred = cfg->get_block_for_node(block->pred(i));
+        worklist.push(pred);
+      }
+    }
+  }
+}
+
+#ifndef PRODUCT
+void XBarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
+  if ((mach->barrier_data() & XLoadBarrierStrong) != 0) {
+    st->print("strong ");
+  }
+  if ((mach->barrier_data() & XLoadBarrierWeak) != 0) {
+    st->print("weak ");
+  }
+  if ((mach->barrier_data() & XLoadBarrierPhantom) != 0) {
+    st->print("phantom ");
+  }
+  if ((mach->barrier_data() & XLoadBarrierNoKeepalive) != 0) {
+    st->print("nokeepalive ");
+  }
+}
+#endif // !PRODUCT
diff --git a/src/hotspot/share/gc/x/c2/xBarrierSetC2.hpp b/src/hotspot/share/gc/x/c2/xBarrierSetC2.hpp
new file mode 100644
index 00000000000..91835338fd7
--- /dev/null
+++ b/src/hotspot/share/gc/x/c2/xBarrierSetC2.hpp
@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_C2_XBARRIERSETC2_HPP
+#define SHARE_GC_X_C2_XBARRIERSETC2_HPP
+
+#include "gc/shared/c2/barrierSetC2.hpp"
+#include "memory/allocation.hpp"
+#include "opto/node.hpp"
+#include "utilities/growableArray.hpp"
+
+const uint8_t XLoadBarrierElided      = 0;
+const uint8_t XLoadBarrierStrong      = 1;
+const uint8_t XLoadBarrierWeak        = 2;
+const uint8_t XLoadBarrierPhantom     = 4;
+const uint8_t XLoadBarrierNoKeepalive = 8;
+
+class XLoadBarrierStubC2 : public ArenaObj {
+private:
+  const MachNode* _node;
+  const Address   _ref_addr;
+  const Register  _ref;
+  const Register  _tmp;
+  const uint8_t   _barrier_data;
+  Label           _entry;
+  Label           _continuation;
+
+  XLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data);
+
+public:
+  static XLoadBarrierStubC2* create(const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data);
+
+  Address ref_addr() const;
+  Register ref() const;
+  Register tmp() const;
+  address slow_path() const;
+  RegMask& live() const;
+  Label* entry();
+  Label* continuation();
+};
+
+class XBarrierSetC2 : public BarrierSetC2 {
+private:
+  void compute_liveness_at_stubs() const;
+  void analyze_dominating_barriers() const;
+
+protected:
+  virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const;
+  virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access,
+                                               Node* expected_val,
+                                               Node* new_val,
+                                               const Type* val_type) const;
+  virtual Node* atomic_cmpxchg_bool_at_resolved(C2AtomicParseAccess& access,
+                                                Node* expected_val,
+                                                Node* new_val,
+                                                const Type* value_type) const;
+  virtual Node* atomic_xchg_at_resolved(C2AtomicParseAccess& access,
+                                        Node* new_val,
+                                        const Type* val_type) const;
+
+public:
+  virtual void* create_barrier_state(Arena* comp_arena) const;
+  virtual bool array_copy_requires_gc_barriers(bool tightly_coupled_alloc,
+                                               BasicType type,
+                                               bool is_clone,
+                                               bool is_clone_instance,
+                                               ArrayCopyPhase phase) const;
+  virtual void clone_at_expansion(PhaseMacroExpand* phase,
+                                  ArrayCopyNode* ac) const;
+
+  virtual void late_barrier_analysis() const;
+  virtual int estimate_stub_size() const;
+  virtual void emit_stubs(CodeBuffer& cb) const;
+
+#ifndef PRODUCT
+  virtual void dump_barrier_data(const MachNode* mach, outputStream* st) const;
+#endif
+};
+
+#endif // SHARE_GC_X_C2_XBARRIERSETC2_HPP
diff --git a/src/hotspot/share/gc/x/vmStructs_x.cpp b/src/hotspot/share/gc/x/vmStructs_x.cpp
new file mode 100644
index 00000000000..4c7d63f41b4
--- /dev/null
+++ b/src/hotspot/share/gc/x/vmStructs_x.cpp
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/vmStructs_x.hpp"
+
+XGlobalsForVMStructs::XGlobalsForVMStructs() :
+    _XGlobalPhase(&XGlobalPhase),
+    _XGlobalSeqNum(&XGlobalSeqNum),
+    _XAddressOffsetMask(&XAddressOffsetMask),
+    _XAddressMetadataMask(&XAddressMetadataMask),
+    _XAddressMetadataFinalizable(&XAddressMetadataFinalizable),
+    _XAddressGoodMask(&XAddressGoodMask),
+    _XAddressBadMask(&XAddressBadMask),
+    _XAddressWeakBadMask(&XAddressWeakBadMask),
+    _XObjectAlignmentSmallShift(&XObjectAlignmentSmallShift),
+    _XObjectAlignmentSmall(&XObjectAlignmentSmall) {
+}
+
+XGlobalsForVMStructs XGlobalsForVMStructs::_instance;
+XGlobalsForVMStructs* XGlobalsForVMStructs::_instance_p = &XGlobalsForVMStructs::_instance;
diff --git a/src/hotspot/share/gc/x/vmStructs_x.hpp b/src/hotspot/share/gc/x/vmStructs_x.hpp
new file mode 100644
index 00000000000..b911c21be23
--- /dev/null
+++ b/src/hotspot/share/gc/x/vmStructs_x.hpp
@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_VMSTRUCTS_X_HPP
+#define SHARE_GC_X_VMSTRUCTS_X_HPP
+
+#include "gc/x/xAttachedArray.hpp"
+#include "gc/x/xCollectedHeap.hpp"
+#include "gc/x/xForwarding.hpp"
+#include "gc/x/xGranuleMap.hpp"
+#include "gc/x/xHeap.hpp"
+#include "gc/x/xPageAllocator.hpp"
+#include "utilities/macros.hpp"
+
+// Expose some ZGC globals to the SA agent.
+class XGlobalsForVMStructs {
+  static XGlobalsForVMStructs _instance;
+
+public:
+  static XGlobalsForVMStructs* _instance_p;
+
+  XGlobalsForVMStructs();
+
+  uint32_t* _XGlobalPhase;
+
+  uint32_t* _XGlobalSeqNum;
+
+  uintptr_t* _XAddressOffsetMask;
+  uintptr_t* _XAddressMetadataMask;
+  uintptr_t* _XAddressMetadataFinalizable;
+  uintptr_t* _XAddressGoodMask;
+  uintptr_t* _XAddressBadMask;
+  uintptr_t* _XAddressWeakBadMask;
+
+  const int* _XObjectAlignmentSmallShift;
+  const int* _XObjectAlignmentSmall;
+};
+
+typedef XGranuleMap<XPage*> XGranuleMapForPageTable;
+typedef XGranuleMap<XForwarding*> XGranuleMapForForwarding;
+typedef XAttachedArray<XForwarding, XForwardingEntry> XAttachedArrayForForwarding;
+
+#define VM_STRUCTS_X(nonstatic_field, volatile_nonstatic_field, static_field)                            \
+  static_field(XGlobalsForVMStructs,            _instance_p,          XGlobalsForVMStructs*)             \
+  nonstatic_field(XGlobalsForVMStructs,         _XGlobalPhase,        uint32_t*)                         \
+  nonstatic_field(XGlobalsForVMStructs,         _XGlobalSeqNum,       uint32_t*)                         \
+  nonstatic_field(XGlobalsForVMStructs,         _XAddressOffsetMask,  uintptr_t*)                        \
+  nonstatic_field(XGlobalsForVMStructs,         _XAddressMetadataMask, uintptr_t*)                       \
+  nonstatic_field(XGlobalsForVMStructs,         _XAddressMetadataFinalizable, uintptr_t*)                \
+  nonstatic_field(XGlobalsForVMStructs,         _XAddressGoodMask,    uintptr_t*)                        \
+  nonstatic_field(XGlobalsForVMStructs,         _XAddressBadMask,     uintptr_t*)                        \
+  nonstatic_field(XGlobalsForVMStructs,         _XAddressWeakBadMask, uintptr_t*)                        \
+  nonstatic_field(XGlobalsForVMStructs,         _XObjectAlignmentSmallShift, const int*)                 \
+  nonstatic_field(XGlobalsForVMStructs,         _XObjectAlignmentSmall, const int*)                      \
+                                                                                                         \
+  nonstatic_field(XCollectedHeap,               _heap,                XHeap)                             \
+                                                                                                         \
+  nonstatic_field(XHeap,                        _page_allocator,      XPageAllocator)                    \
+  nonstatic_field(XHeap,                        _page_table,          XPageTable)                        \
+  nonstatic_field(XHeap,                        _forwarding_table,    XForwardingTable)                  \
+  nonstatic_field(XHeap,                        _relocate,            XRelocate)                         \
+                                                                                                         \
+  nonstatic_field(XPage,                        _type,                const uint8_t)                     \
+  nonstatic_field(XPage,                        _seqnum,              uint32_t)                          \
+  nonstatic_field(XPage,                        _virtual,             const XVirtualMemory)              \
+  volatile_nonstatic_field(XPage,               _top,                 uintptr_t)                         \
+                                                                                                         \
+  nonstatic_field(XPageAllocator,               _max_capacity,        const size_t)                      \
+  volatile_nonstatic_field(XPageAllocator,      _capacity,            size_t)                            \
+  volatile_nonstatic_field(XPageAllocator,      _used,                size_t)                            \
+                                                                                                         \
+  nonstatic_field(XPageTable,                   _map,                 XGranuleMapForPageTable)           \
+                                                                                                         \
+  nonstatic_field(XGranuleMapForPageTable,      _map,                 XPage** const)                     \
+  nonstatic_field(XGranuleMapForForwarding,     _map,                 XForwarding** const)               \
+                                                                                                         \
+  nonstatic_field(XForwardingTable,             _map,                 XGranuleMapForForwarding)          \
+                                                                                                         \
+  nonstatic_field(XVirtualMemory,               _start,               const uintptr_t)                   \
+  nonstatic_field(XVirtualMemory,               _end,                 const uintptr_t)                   \
+                                                                                                         \
+  nonstatic_field(XForwarding,                  _virtual,             const XVirtualMemory)              \
+  nonstatic_field(XForwarding,                  _object_alignment_shift, const size_t)                   \
+  volatile_nonstatic_field(XForwarding,         _ref_count,           int)                               \
+  nonstatic_field(XForwarding,                  _entries,             const XAttachedArrayForForwarding) \
+  nonstatic_field(XForwardingEntry,             _entry,               uint64_t)                          \
+  nonstatic_field(XAttachedArrayForForwarding,  _length,              const size_t)
+
+#define VM_INT_CONSTANTS_X(declare_constant, declare_constant_with_value)                                \
+  declare_constant(XPhaseRelocate)                                                                       \
+  declare_constant(XPageTypeSmall)                                                                       \
+  declare_constant(XPageTypeMedium)                                                                      \
+  declare_constant(XPageTypeLarge)                                                                       \
+  declare_constant(XObjectAlignmentMediumShift)                                                          \
+  declare_constant(XObjectAlignmentLargeShift)
+
+#define VM_LONG_CONSTANTS_X(declare_constant)                                                            \
+  declare_constant(XGranuleSizeShift)                                                                    \
+  declare_constant(XPageSizeSmallShift)                                                                  \
+  declare_constant(XPageSizeMediumShift)                                                                 \
+  declare_constant(XAddressOffsetShift)                                                                  \
+  declare_constant(XAddressOffsetBits)                                                                   \
+  declare_constant(XAddressOffsetMask)                                                                   \
+  declare_constant(XAddressOffsetMax)
+
+#define VM_TYPES_X(declare_type, declare_toplevel_type, declare_integer_type)                            \
+  declare_toplevel_type(XGlobalsForVMStructs)                                                            \
+  declare_type(XCollectedHeap, CollectedHeap)                                                            \
+  declare_toplevel_type(XHeap)                                                                           \
+  declare_toplevel_type(XRelocate)                                                                       \
+  declare_toplevel_type(XPage)                                                                           \
+  declare_toplevel_type(XPageAllocator)                                                                  \
+  declare_toplevel_type(XPageTable)                                                                      \
+  declare_toplevel_type(XAttachedArrayForForwarding)                                                     \
+  declare_toplevel_type(XGranuleMapForPageTable)                                                         \
+  declare_toplevel_type(XGranuleMapForForwarding)                                                        \
+  declare_toplevel_type(XVirtualMemory)                                                                  \
+  declare_toplevel_type(XForwardingTable)                                                                \
+  declare_toplevel_type(XForwarding)                                                                     \
+  declare_toplevel_type(XForwardingEntry)                                                                \
+  declare_toplevel_type(XPhysicalMemoryManager)
+
+#endif // SHARE_GC_X_VMSTRUCTS_X_HPP
diff --git a/src/hotspot/share/gc/x/xAbort.cpp b/src/hotspot/share/gc/x/xAbort.cpp
new file mode 100644
index 00000000000..11b8d840d22
--- /dev/null
+++ b/src/hotspot/share/gc/x/xAbort.cpp
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xAbort.hpp"
+#include "runtime/atomic.hpp"
+
+volatile bool XAbort::_should_abort = false;
+
+void XAbort::abort() {
+  Atomic::release_store_fence(&_should_abort, true);
+}
diff --git a/src/hotspot/share/gc/x/xAbort.hpp b/src/hotspot/share/gc/x/xAbort.hpp
new file mode 100644
index 00000000000..808a350584b
--- /dev/null
+++ b/src/hotspot/share/gc/x/xAbort.hpp
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XABORT_HPP
+#define SHARE_GC_X_XABORT_HPP
+
+#include "memory/allStatic.hpp"
+
+class XAbort : public AllStatic {
+private:
+  static volatile bool _should_abort;
+
+public:
+  static bool should_abort();
+  static void abort();
+};
+
+#endif // SHARE_GC_X_XABORT_HPP
diff --git a/src/hotspot/share/gc/x/xAbort.inline.hpp b/src/hotspot/share/gc/x/xAbort.inline.hpp
new file mode 100644
index 00000000000..8ef1219330a
--- /dev/null
+++ b/src/hotspot/share/gc/x/xAbort.inline.hpp
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XABORT_INLINE_HPP
+#define SHARE_GC_X_XABORT_INLINE_HPP
+
+#include "gc/x/xAbort.hpp"
+
+#include "runtime/atomic.hpp"
+
+inline bool XAbort::should_abort() {
+  return Atomic::load_acquire(&_should_abort);
+}
+
+#endif // SHARE_GC_X_XABORT_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xAddress.cpp b/src/hotspot/share/gc/x/xAddress.cpp
new file mode 100644
index 00000000000..33dffc662f1
--- /dev/null
+++ b/src/hotspot/share/gc/x/xAddress.cpp
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xAddress.hpp"
+#include "gc/x/xGlobals.hpp"
+
+void XAddress::set_good_mask(uintptr_t mask) {
+  XAddressGoodMask = mask;
+  XAddressBadMask = XAddressGoodMask ^ XAddressMetadataMask;
+  XAddressWeakBadMask = (XAddressGoodMask | XAddressMetadataRemapped | XAddressMetadataFinalizable) ^ XAddressMetadataMask;
+}
+
+void XAddress::initialize() {
+  XAddressOffsetBits = XPlatformAddressOffsetBits();
+  XAddressOffsetMask = (((uintptr_t)1 << XAddressOffsetBits) - 1) << XAddressOffsetShift;
+  XAddressOffsetMax = (uintptr_t)1 << XAddressOffsetBits;
+
+  XAddressMetadataShift = XPlatformAddressMetadataShift();
+  XAddressMetadataMask = (((uintptr_t)1 << XAddressMetadataBits) - 1) << XAddressMetadataShift;
+
+  XAddressMetadataMarked0 = (uintptr_t)1 << (XAddressMetadataShift + 0);
+  XAddressMetadataMarked1 = (uintptr_t)1 << (XAddressMetadataShift + 1);
+  XAddressMetadataRemapped = (uintptr_t)1 << (XAddressMetadataShift + 2);
+  XAddressMetadataFinalizable = (uintptr_t)1 << (XAddressMetadataShift + 3);
+
+  XAddressMetadataMarked = XAddressMetadataMarked0;
+  set_good_mask(XAddressMetadataRemapped);
+}
+
+void XAddress::flip_to_marked() {
+  XAddressMetadataMarked ^= (XAddressMetadataMarked0 | XAddressMetadataMarked1);
+  set_good_mask(XAddressMetadataMarked);
+}
+
+void XAddress::flip_to_remapped() {
+  set_good_mask(XAddressMetadataRemapped);
+}
diff --git a/src/hotspot/share/gc/x/xAddress.hpp b/src/hotspot/share/gc/x/xAddress.hpp
new file mode 100644
index 00000000000..ff9d548f1af
--- /dev/null
+++ b/src/hotspot/share/gc/x/xAddress.hpp
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XADDRESS_HPP
+#define SHARE_GC_X_XADDRESS_HPP
+
+#include "memory/allStatic.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class XAddress : public AllStatic {
+  friend class XAddressTest;
+
+private:
+  static void set_good_mask(uintptr_t mask);
+
+public:
+  static void initialize();
+
+  static void flip_to_marked();
+  static void flip_to_remapped();
+
+  static bool is_null(uintptr_t value);
+  static bool is_bad(uintptr_t value);
+  static bool is_good(uintptr_t value);
+  static bool is_good_or_null(uintptr_t value);
+  static bool is_weak_bad(uintptr_t value);
+  static bool is_weak_good(uintptr_t value);
+  static bool is_weak_good_or_null(uintptr_t value);
+  static bool is_marked(uintptr_t value);
+  static bool is_marked_or_null(uintptr_t value);
+  static bool is_finalizable(uintptr_t value);
+  static bool is_finalizable_good(uintptr_t value);
+  static bool is_remapped(uintptr_t value);
+  static bool is_in(uintptr_t value);
+
+  static uintptr_t offset(uintptr_t value);
+  static uintptr_t good(uintptr_t value);
+  static uintptr_t good_or_null(uintptr_t value);
+  static uintptr_t finalizable_good(uintptr_t value);
+  static uintptr_t marked(uintptr_t value);
+  static uintptr_t marked0(uintptr_t value);
+  static uintptr_t marked1(uintptr_t value);
+  static uintptr_t remapped(uintptr_t value);
+  static uintptr_t remapped_or_null(uintptr_t value);
+};
+
+#endif // SHARE_GC_X_XADDRESS_HPP
diff --git a/src/hotspot/share/gc/x/xAddress.inline.hpp b/src/hotspot/share/gc/x/xAddress.inline.hpp
new file mode 100644
index 00000000000..046ee10af00
--- /dev/null
+++ b/src/hotspot/share/gc/x/xAddress.inline.hpp
@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XADDRESS_INLINE_HPP
+#define SHARE_GC_X_XADDRESS_INLINE_HPP
+
+#include "gc/x/xAddress.hpp"
+
+#include "gc/x/xGlobals.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/macros.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+inline bool XAddress::is_null(uintptr_t value) {
+  return value == 0;
+}
+
+inline bool XAddress::is_bad(uintptr_t value) {
+  return value & XAddressBadMask;
+}
+
+inline bool XAddress::is_good(uintptr_t value) {
+  return !is_bad(value) && !is_null(value);
+}
+
+inline bool XAddress::is_good_or_null(uintptr_t value) {
+  // Checking if an address is "not bad" is an optimized version of
+  // checking if it's "good or null", which eliminates an explicit
+  // null check. However, the implicit null check only checks that
+  // the mask bits are zero, not that the entire address is zero.
+  // This means that an address without mask bits would pass through
+  // the barrier as if it was null. This should be harmless as such
+  // addresses should ever be passed through the barrier.
+  const bool result = !is_bad(value);
+  assert((is_good(value) || is_null(value)) == result, "Bad address");
+  return result;
+}
+
+inline bool XAddress::is_weak_bad(uintptr_t value) {
+  return value & XAddressWeakBadMask;
+}
+
+inline bool XAddress::is_weak_good(uintptr_t value) {
+  return !is_weak_bad(value) && !is_null(value);
+}
+
+inline bool XAddress::is_weak_good_or_null(uintptr_t value) {
+  return !is_weak_bad(value);
+}
+
+inline bool XAddress::is_marked(uintptr_t value) {
+  return value & XAddressMetadataMarked;
+}
+
+inline bool XAddress::is_marked_or_null(uintptr_t value) {
+  return is_marked(value) || is_null(value);
+}
+
+inline bool XAddress::is_finalizable(uintptr_t value) {
+  return value & XAddressMetadataFinalizable;
+}
+
+inline bool XAddress::is_finalizable_good(uintptr_t value) {
+  return is_finalizable(value) && is_good(value ^ XAddressMetadataFinalizable);
+}
+
+inline bool XAddress::is_remapped(uintptr_t value) {
+  return value & XAddressMetadataRemapped;
+}
+
+inline bool XAddress::is_in(uintptr_t value) {
+  // Check that exactly one non-offset bit is set
+  if (!is_power_of_2(value & ~XAddressOffsetMask)) {
+    return false;
+  }
+
+  // Check that one of the non-finalizable metadata is set
+  return value & (XAddressMetadataMask & ~XAddressMetadataFinalizable);
+}
+
+inline uintptr_t XAddress::offset(uintptr_t value) {
+  return value & XAddressOffsetMask;
+}
+
+inline uintptr_t XAddress::good(uintptr_t value) {
+  return offset(value) | XAddressGoodMask;
+}
+
+inline uintptr_t XAddress::good_or_null(uintptr_t value) {
+  return is_null(value) ? 0 : good(value);
+}
+
+inline uintptr_t XAddress::finalizable_good(uintptr_t value) {
+  return offset(value) | XAddressMetadataFinalizable | XAddressGoodMask;
+}
+
+inline uintptr_t XAddress::marked(uintptr_t value) {
+  return offset(value) | XAddressMetadataMarked;
+}
+
+inline uintptr_t XAddress::marked0(uintptr_t value) {
+  return offset(value) | XAddressMetadataMarked0;
+}
+
+inline uintptr_t XAddress::marked1(uintptr_t value) {
+  return offset(value) | XAddressMetadataMarked1;
+}
+
+inline uintptr_t XAddress::remapped(uintptr_t value) {
+  return offset(value) | XAddressMetadataRemapped;
+}
+
+inline uintptr_t XAddress::remapped_or_null(uintptr_t value) {
+  return is_null(value) ? 0 : remapped(value);
+}
+
+#endif // SHARE_GC_X_XADDRESS_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xAddressSpaceLimit.cpp b/src/hotspot/share/gc/x/xAddressSpaceLimit.cpp
new file mode 100644
index 00000000000..6d3c7a295df
--- /dev/null
+++ b/src/hotspot/share/gc/x/xAddressSpaceLimit.cpp
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/xAddressSpaceLimit.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/os.hpp"
+#include "utilities/align.hpp"
+
+static size_t address_space_limit() {
+  size_t limit = 0;
+
+  if (os::has_allocatable_memory_limit(&limit)) {
+    return limit;
+  }
+
+  // No limit
+  return SIZE_MAX;
+}
+
+size_t XAddressSpaceLimit::mark_stack() {
+  // Allow mark stacks to occupy 10% of the address space
+  const size_t limit = address_space_limit() / 10;
+  return align_up(limit, XMarkStackSpaceExpandSize);
+}
+
+size_t XAddressSpaceLimit::heap_view() {
+  // Allow all heap views to occupy 50% of the address space
+  const size_t limit = address_space_limit() / MaxVirtMemFraction / XHeapViews;
+  return align_up(limit, XGranuleSize);
+}
diff --git a/src/hotspot/share/gc/x/xAddressSpaceLimit.hpp b/src/hotspot/share/gc/x/xAddressSpaceLimit.hpp
new file mode 100644
index 00000000000..9a3fcc27a29
--- /dev/null
+++ b/src/hotspot/share/gc/x/xAddressSpaceLimit.hpp
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XADDRESSSPACELIMIT_HPP
+#define SHARE_GC_X_XADDRESSSPACELIMIT_HPP
+
+#include "memory/allStatic.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class XAddressSpaceLimit : public AllStatic {
+public:
+  static size_t mark_stack();
+  static size_t heap_view();
+};
+
+#endif // SHARE_GC_X_XADDRESSSPACELIMIT_HPP
diff --git a/src/hotspot/share/gc/x/xAllocationFlags.hpp b/src/hotspot/share/gc/x/xAllocationFlags.hpp
new file mode 100644
index 00000000000..307d68c65ac
--- /dev/null
+++ b/src/hotspot/share/gc/x/xAllocationFlags.hpp
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XALLOCATIONFLAGS_HPP
+#define SHARE_GC_X_XALLOCATIONFLAGS_HPP
+
+#include "gc/x/xBitField.hpp"
+#include "memory/allocation.hpp"
+
+//
+// Allocation flags layout
+// -----------------------
+//
+//   7     2 1 0
+//  +-----+-+-+-+
+//  |00000|1|1|1|
+//  +-----+-+-+-+
+//  |     | | |
+//  |     | | * 0-0 Non-Blocking Flag (1-bit)
+//  |     | |
+//  |     | * 1-1 Worker Relocation Flag (1-bit)
+//  |     |
+//  |     * 2-2 Low Address Flag (1-bit)
+//  |
+//  * 7-3 Unused (5-bits)
+//
+
+class XAllocationFlags {
+private:
+  typedef XBitField<uint8_t, bool, 0, 1> field_non_blocking;
+  typedef XBitField<uint8_t, bool, 1, 1> field_worker_relocation;
+  typedef XBitField<uint8_t, bool, 2, 1> field_low_address;
+
+  uint8_t _flags;
+
+public:
+  XAllocationFlags() :
+      _flags(0) {}
+
+  void set_non_blocking() {
+    _flags |= field_non_blocking::encode(true);
+  }
+
+  void set_worker_relocation() {
+    _flags |= field_worker_relocation::encode(true);
+  }
+
+  void set_low_address() {
+    _flags |= field_low_address::encode(true);
+  }
+
+  bool non_blocking() const {
+    return field_non_blocking::decode(_flags);
+  }
+
+  bool worker_relocation() const {
+    return field_worker_relocation::decode(_flags);
+  }
+
+  bool low_address() const {
+    return field_low_address::decode(_flags);
+  }
+};
+
+#endif // SHARE_GC_X_XALLOCATIONFLAGS_HPP
diff --git a/src/hotspot/share/gc/x/xArguments.cpp b/src/hotspot/share/gc/x/xArguments.cpp
new file mode 100644
index 00000000000..8c02c800247
--- /dev/null
+++ b/src/hotspot/share/gc/x/xArguments.cpp
@@ -0,0 +1,123 @@
+/*
+ * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xAddressSpaceLimit.hpp"
+#include "gc/x/xArguments.hpp"
+#include "gc/x/xCollectedHeap.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xHeuristics.hpp"
+#include "gc/shared/gcArguments.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/globals_extension.hpp"
+#include "runtime/java.hpp"
+
+void XArguments::initialize_alignments() {
+  SpaceAlignment = XGranuleSize;
+  HeapAlignment = SpaceAlignment;
+}
+
+void XArguments::initialize() {
+  // Check mark stack size
+  const size_t mark_stack_space_limit = XAddressSpaceLimit::mark_stack();
+  if (ZMarkStackSpaceLimit > mark_stack_space_limit) {
+    if (!FLAG_IS_DEFAULT(ZMarkStackSpaceLimit)) {
+      vm_exit_during_initialization("ZMarkStackSpaceLimit too large for limited address space");
+    }
+    FLAG_SET_DEFAULT(ZMarkStackSpaceLimit, mark_stack_space_limit);
+  }
+
+  // Enable NUMA by default
+  if (FLAG_IS_DEFAULT(UseNUMA)) {
+    FLAG_SET_DEFAULT(UseNUMA, true);
+  }
+
+  if (FLAG_IS_DEFAULT(ZFragmentationLimit)) {
+    FLAG_SET_DEFAULT(ZFragmentationLimit, 25.0);
+  }
+
+  // Select number of parallel threads
+  if (FLAG_IS_DEFAULT(ParallelGCThreads)) {
+    FLAG_SET_DEFAULT(ParallelGCThreads, XHeuristics::nparallel_workers());
+  }
+
+  if (ParallelGCThreads == 0) {
+    vm_exit_during_initialization("The flag -XX:+UseZGC can not be combined with -XX:ParallelGCThreads=0");
+  }
+
+  // Select number of concurrent threads
+  if (FLAG_IS_DEFAULT(ConcGCThreads)) {
+    FLAG_SET_DEFAULT(ConcGCThreads, XHeuristics::nconcurrent_workers());
+  }
+
+  if (ConcGCThreads == 0) {
+    vm_exit_during_initialization("The flag -XX:+UseZGC can not be combined with -XX:ConcGCThreads=0");
+  }
+
+  // Large page size must match granule size
+  if (!FLAG_IS_DEFAULT(LargePageSizeInBytes) && LargePageSizeInBytes != XGranuleSize) {
+    vm_exit_during_initialization(err_msg("Incompatible -XX:LargePageSizeInBytes, only "
+                                          SIZE_FORMAT "M large pages are supported by ZGC",
+                                          XGranuleSize / M));
+  }
+
+  // The heuristics used when UseDynamicNumberOfGCThreads is
+  // enabled defaults to using a ZAllocationSpikeTolerance of 1.
+  if (UseDynamicNumberOfGCThreads && FLAG_IS_DEFAULT(ZAllocationSpikeTolerance)) {
+    FLAG_SET_DEFAULT(ZAllocationSpikeTolerance, 1);
+  }
+
+#ifdef COMPILER2
+  // Enable loop strip mining by default
+  if (FLAG_IS_DEFAULT(UseCountedLoopSafepoints)) {
+    FLAG_SET_DEFAULT(UseCountedLoopSafepoints, true);
+    if (FLAG_IS_DEFAULT(LoopStripMiningIter)) {
+      FLAG_SET_DEFAULT(LoopStripMiningIter, 1000);
+    }
+  }
+#endif
+
+  // CompressedOops not supported
+  FLAG_SET_DEFAULT(UseCompressedOops, false);
+
+  // Verification before startup and after exit not (yet) supported
+  FLAG_SET_DEFAULT(VerifyDuringStartup, false);
+  FLAG_SET_DEFAULT(VerifyBeforeExit, false);
+
+  if (VerifyBeforeGC || VerifyDuringGC || VerifyAfterGC) {
+    FLAG_SET_DEFAULT(ZVerifyRoots, true);
+    FLAG_SET_DEFAULT(ZVerifyObjects, true);
+  }
+}
+
+size_t XArguments::heap_virtual_to_physical_ratio() {
+  return XHeapViews * XVirtualToPhysicalRatio;
+}
+
+CollectedHeap* XArguments::create_heap() {
+  return new XCollectedHeap();
+}
+
+bool XArguments::is_supported() {
+  return is_os_supported();
+}
diff --git a/src/hotspot/share/gc/x/xArguments.hpp b/src/hotspot/share/gc/x/xArguments.hpp
new file mode 100644
index 00000000000..aaa586a2df2
--- /dev/null
+++ b/src/hotspot/share/gc/x/xArguments.hpp
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XARGUMENTS_HPP
+#define SHARE_GC_X_XARGUMENTS_HPP
+
+#include "gc/shared/gcArguments.hpp"
+
+class CollectedHeap;
+
+class XArguments : AllStatic {
+public:
+  static void initialize_alignments();
+  static void initialize();
+  static size_t heap_virtual_to_physical_ratio();
+  static CollectedHeap* create_heap();
+
+  static bool is_supported();
+
+  static bool is_os_supported();
+};
+
+#endif // SHARE_GC_X_XARGUMENTS_HPP
diff --git a/src/hotspot/share/gc/x/xArray.hpp b/src/hotspot/share/gc/x/xArray.hpp
new file mode 100644
index 00000000000..b0b4b5bd81e
--- /dev/null
+++ b/src/hotspot/share/gc/x/xArray.hpp
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XARRAY_HPP
+#define SHARE_GC_X_XARRAY_HPP
+
+#include "memory/allocation.hpp"
+#include "utilities/growableArray.hpp"
+
+template <typename T> using XArray = GrowableArrayCHeap<T, mtGC>;
+
+template <typename T, bool Parallel>
+class XArrayIteratorImpl : public StackObj {
+private:
+  const T*       _next;
+  const T* const _end;
+
+  bool next_serial(T* elem);
+  bool next_parallel(T* elem);
+
+public:
+  XArrayIteratorImpl(const T* array, size_t length);
+  XArrayIteratorImpl(const XArray<T>* array);
+
+  bool next(T* elem);
+};
+
+template <typename T> using XArrayIterator = XArrayIteratorImpl<T, false /* Parallel */>;
+template <typename T> using XArrayParallelIterator = XArrayIteratorImpl<T, true /* Parallel */>;
+
+#endif // SHARE_GC_X_XARRAY_HPP
diff --git a/src/hotspot/share/gc/x/xArray.inline.hpp b/src/hotspot/share/gc/x/xArray.inline.hpp
new file mode 100644
index 00000000000..9d3cfcfbc65
--- /dev/null
+++ b/src/hotspot/share/gc/x/xArray.inline.hpp
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XARRAY_INLINE_HPP
+#define SHARE_GC_X_XARRAY_INLINE_HPP
+
+#include "gc/x/xArray.hpp"
+
+#include "runtime/atomic.hpp"
+
+template <typename T, bool Parallel>
+inline bool XArrayIteratorImpl<T, Parallel>::next_serial(T* elem) {
+  if (_next == _end) {
+    return false;
+  }
+
+  *elem = *_next;
+  _next++;
+
+  return true;
+}
+
+template <typename T, bool Parallel>
+inline bool XArrayIteratorImpl<T, Parallel>::next_parallel(T* elem) {
+  const T* old_next = Atomic::load(&_next);
+
+  for (;;) {
+    if (old_next == _end) {
+      return false;
+    }
+
+    const T* const new_next = old_next + 1;
+    const T* const prev_next = Atomic::cmpxchg(&_next, old_next, new_next);
+    if (prev_next == old_next) {
+      *elem = *old_next;
+      return true;
+    }
+
+    old_next = prev_next;
+  }
+}
+
+template <typename T, bool Parallel>
+inline XArrayIteratorImpl<T, Parallel>::XArrayIteratorImpl(const T* array, size_t length) :
+    _next(array),
+    _end(array + length) {}
+
+template <typename T, bool Parallel>
+inline XArrayIteratorImpl<T, Parallel>::XArrayIteratorImpl(const XArray<T>* array) :
+    XArrayIteratorImpl<T, Parallel>(array->is_empty() ? NULL : array->adr_at(0), array->length()) {}
+
+template <typename T, bool Parallel>
+inline bool XArrayIteratorImpl<T, Parallel>::next(T* elem) {
+  if (Parallel) {
+    return next_parallel(elem);
+  } else {
+    return next_serial(elem);
+  }
+}
+
+#endif // SHARE_GC_X_XARRAY_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xAttachedArray.hpp b/src/hotspot/share/gc/x/xAttachedArray.hpp
new file mode 100644
index 00000000000..f039f602aab
--- /dev/null
+++ b/src/hotspot/share/gc/x/xAttachedArray.hpp
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XATTACHEDARRAY_HPP
+#define SHARE_GC_X_XATTACHEDARRAY_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+class VMStructs;
+
+template <typename ObjectT, typename ArrayT>
+class XAttachedArray {
+  friend class ::VMStructs;
+
+private:
+  const size_t _length;
+
+  static size_t object_size();
+  static size_t array_size(size_t length);
+
+public:
+  template <typename Allocator>
+  static void* alloc(Allocator* allocator, size_t length);
+
+  static void* alloc(size_t length);
+  static void free(ObjectT* obj);
+
+  XAttachedArray(size_t length);
+
+  size_t length() const;
+  ArrayT* operator()(const ObjectT* obj) const;
+};
+
+#endif // SHARE_GC_X_XATTACHEDARRAY_HPP
diff --git a/src/hotspot/share/gc/x/xAttachedArray.inline.hpp b/src/hotspot/share/gc/x/xAttachedArray.inline.hpp
new file mode 100644
index 00000000000..ba10de99673
--- /dev/null
+++ b/src/hotspot/share/gc/x/xAttachedArray.inline.hpp
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XATTACHEDARRAY_INLINE_HPP
+#define SHARE_GC_X_XATTACHEDARRAY_INLINE_HPP
+
+#include "gc/x/xAttachedArray.hpp"
+
+#include "memory/allocation.hpp"
+#include "utilities/align.hpp"
+
+template <typename ObjectT, typename ArrayT>
+inline size_t XAttachedArray<ObjectT, ArrayT>::object_size() {
+  return align_up(sizeof(ObjectT), sizeof(ArrayT));
+}
+
+template <typename ObjectT, typename ArrayT>
+inline size_t XAttachedArray<ObjectT, ArrayT>::array_size(size_t length) {
+  return sizeof(ArrayT) * length;
+}
+
+template <typename ObjectT, typename ArrayT>
+template <typename Allocator>
+inline void* XAttachedArray<ObjectT, ArrayT>::alloc(Allocator* allocator, size_t length) {
+  // Allocate memory for object and array
+  const size_t size = object_size() + array_size(length);
+  void* const addr = allocator->alloc(size);
+
+  // Placement new array
+  void* const array_addr = reinterpret_cast<char*>(addr) + object_size();
+  ::new (array_addr) ArrayT[length];
+
+  // Return pointer to object
+  return addr;
+}
+
+template <typename ObjectT, typename ArrayT>
+inline void* XAttachedArray<ObjectT, ArrayT>::alloc(size_t length) {
+  struct Allocator {
+    void* alloc(size_t size) const {
+      return AllocateHeap(size, mtGC);
+    }
+  } allocator;
+  return alloc(&allocator, length);
+}
+
+template <typename ObjectT, typename ArrayT>
+inline void XAttachedArray<ObjectT, ArrayT>::free(ObjectT* obj) {
+  FreeHeap(obj);
+}
+
+template <typename ObjectT, typename ArrayT>
+inline XAttachedArray<ObjectT, ArrayT>::XAttachedArray(size_t length) :
+    _length(length) {}
+
+template <typename ObjectT, typename ArrayT>
+inline size_t XAttachedArray<ObjectT, ArrayT>::length() const {
+  return _length;
+}
+
+template <typename ObjectT, typename ArrayT>
+inline ArrayT* XAttachedArray<ObjectT, ArrayT>::operator()(const ObjectT* obj) const {
+  return reinterpret_cast<ArrayT*>(reinterpret_cast<uintptr_t>(obj) + object_size());
+}
+
+#endif // SHARE_GC_X_XATTACHEDARRAY_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xBarrier.cpp b/src/hotspot/share/gc/x/xBarrier.cpp
new file mode 100644
index 00000000000..a2528a9aaf2
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBarrier.cpp
@@ -0,0 +1,275 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "classfile/javaClasses.hpp"
+#include "gc/x/xBarrier.inline.hpp"
+#include "gc/x/xHeap.inline.hpp"
+#include "gc/x/xOop.inline.hpp"
+#include "gc/x/xThread.inline.hpp"
+#include "memory/iterator.inline.hpp"
+#include "oops/oop.inline.hpp"
+#include "runtime/safepoint.hpp"
+#include "utilities/debug.hpp"
+
+template <bool finalizable>
+bool XBarrier::should_mark_through(uintptr_t addr) {
+  // Finalizable marked oops can still exists on the heap after marking
+  // has completed, in which case we just want to convert this into a
+  // good oop and not push it on the mark stack.
+  if (!during_mark()) {
+    assert(XAddress::is_marked(addr), "Should be marked");
+    assert(XAddress::is_finalizable(addr), "Should be finalizable");
+    return false;
+  }
+
+  // During marking, we mark through already marked oops to avoid having
+  // some large part of the object graph hidden behind a pushed, but not
+  // yet flushed, entry on a mutator mark stack. Always marking through
+  // allows the GC workers to proceed through the object graph even if a
+  // mutator touched an oop first, which in turn will reduce the risk of
+  // having to flush mark stacks multiple times to terminate marking.
+  //
+  // However, when doing finalizable marking we don't always want to mark
+  // through. First, marking through an already strongly marked oop would
+  // be wasteful, since we will then proceed to do finalizable marking on
+  // an object which is, or will be, marked strongly. Second, marking
+  // through an already finalizable marked oop would also be wasteful,
+  // since such oops can never end up on a mutator mark stack and can
+  // therefore not hide some part of the object graph from GC workers.
+  if (finalizable) {
+    return !XAddress::is_marked(addr);
+  }
+
+  // Mark through
+  return true;
+}
+
+template <bool gc_thread, bool follow, bool finalizable, bool publish>
+uintptr_t XBarrier::mark(uintptr_t addr) {
+  uintptr_t good_addr;
+
+  if (XAddress::is_marked(addr)) {
+    // Already marked, but try to mark though anyway
+    good_addr = XAddress::good(addr);
+  } else if (XAddress::is_remapped(addr)) {
+    // Already remapped, but also needs to be marked
+    good_addr = XAddress::good(addr);
+  } else {
+    // Needs to be both remapped and marked
+    good_addr = remap(addr);
+  }
+
+  // Mark
+  if (should_mark_through<finalizable>(addr)) {
+    XHeap::heap()->mark_object<gc_thread, follow, finalizable, publish>(good_addr);
+  }
+
+  if (finalizable) {
+    // Make the oop finalizable marked/good, instead of normal marked/good.
+    // This is needed because an object might first becomes finalizable
+    // marked by the GC, and then loaded by a mutator thread. In this case,
+    // the mutator thread must be able to tell that the object needs to be
+    // strongly marked. The finalizable bit in the oop exists to make sure
+    // that a load of a finalizable marked oop will fall into the barrier
+    // slow path so that we can mark the object as strongly reachable.
+    return XAddress::finalizable_good(good_addr);
+  }
+
+  return good_addr;
+}
+
+uintptr_t XBarrier::remap(uintptr_t addr) {
+  assert(!XAddress::is_good(addr), "Should not be good");
+  assert(!XAddress::is_weak_good(addr), "Should not be weak good");
+  return XHeap::heap()->remap_object(addr);
+}
+
+uintptr_t XBarrier::relocate(uintptr_t addr) {
+  assert(!XAddress::is_good(addr), "Should not be good");
+  assert(!XAddress::is_weak_good(addr), "Should not be weak good");
+  return XHeap::heap()->relocate_object(addr);
+}
+
+uintptr_t XBarrier::relocate_or_mark(uintptr_t addr) {
+  return during_relocate() ? relocate(addr) : mark<AnyThread, Follow, Strong, Publish>(addr);
+}
+
+uintptr_t XBarrier::relocate_or_mark_no_follow(uintptr_t addr) {
+  return during_relocate() ? relocate(addr) : mark<AnyThread, DontFollow, Strong, Publish>(addr);
+}
+
+uintptr_t XBarrier::relocate_or_remap(uintptr_t addr) {
+  return during_relocate() ? relocate(addr) : remap(addr);
+}
+
+//
+// Load barrier
+//
+uintptr_t XBarrier::load_barrier_on_oop_slow_path(uintptr_t addr) {
+  return relocate_or_mark(addr);
+}
+
+uintptr_t XBarrier::load_barrier_on_invisible_root_oop_slow_path(uintptr_t addr) {
+  return relocate_or_mark_no_follow(addr);
+}
+
+void XBarrier::load_barrier_on_oop_fields(oop o) {
+  assert(XAddress::is_good(XOop::to_address(o)), "Should be good");
+  XLoadBarrierOopClosure cl;
+  o->oop_iterate(&cl);
+}
+
+//
+// Weak load barrier
+//
+uintptr_t XBarrier::weak_load_barrier_on_oop_slow_path(uintptr_t addr) {
+  return XAddress::is_weak_good(addr) ? XAddress::good(addr) : relocate_or_remap(addr);
+}
+
+uintptr_t XBarrier::weak_load_barrier_on_weak_oop_slow_path(uintptr_t addr) {
+  const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr);
+  if (XHeap::heap()->is_object_strongly_live(good_addr)) {
+    return good_addr;
+  }
+
+  // Not strongly live
+  return 0;
+}
+
+uintptr_t XBarrier::weak_load_barrier_on_phantom_oop_slow_path(uintptr_t addr) {
+  const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr);
+  if (XHeap::heap()->is_object_live(good_addr)) {
+    return good_addr;
+  }
+
+  // Not live
+  return 0;
+}
+
+//
+// Keep alive barrier
+//
+uintptr_t XBarrier::keep_alive_barrier_on_oop_slow_path(uintptr_t addr) {
+  assert(during_mark(), "Invalid phase");
+
+  // Mark
+  return mark<AnyThread, Follow, Strong, Overflow>(addr);
+}
+
+uintptr_t XBarrier::keep_alive_barrier_on_weak_oop_slow_path(uintptr_t addr) {
+  assert(XResurrection::is_blocked(), "This operation is only valid when resurrection is blocked");
+  const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr);
+  assert(XHeap::heap()->is_object_strongly_live(good_addr), "Should be live");
+  return good_addr;
+}
+
+uintptr_t XBarrier::keep_alive_barrier_on_phantom_oop_slow_path(uintptr_t addr) {
+  assert(XResurrection::is_blocked(), "This operation is only valid when resurrection is blocked");
+  const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr);
+  assert(XHeap::heap()->is_object_live(good_addr), "Should be live");
+  return good_addr;
+}
+
+//
+// Mark barrier
+//
+uintptr_t XBarrier::mark_barrier_on_oop_slow_path(uintptr_t addr) {
+  assert(during_mark(), "Invalid phase");
+  assert(XThread::is_worker(), "Invalid thread");
+
+  // Mark
+  return mark<GCThread, Follow, Strong, Overflow>(addr);
+}
+
+uintptr_t XBarrier::mark_barrier_on_finalizable_oop_slow_path(uintptr_t addr) {
+  assert(during_mark(), "Invalid phase");
+  assert(XThread::is_worker(), "Invalid thread");
+
+  // Mark
+  return mark<GCThread, Follow, Finalizable, Overflow>(addr);
+}
+
+//
+// Narrow oop variants, never used.
+//
+oop XBarrier::load_barrier_on_oop_field(volatile narrowOop* p) {
+  ShouldNotReachHere();
+  return NULL;
+}
+
+oop XBarrier::load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o) {
+  ShouldNotReachHere();
+  return NULL;
+}
+
+void XBarrier::load_barrier_on_oop_array(volatile narrowOop* p, size_t length) {
+  ShouldNotReachHere();
+}
+
+oop XBarrier::load_barrier_on_weak_oop_field_preloaded(volatile narrowOop* p, oop o) {
+  ShouldNotReachHere();
+  return NULL;
+}
+
+oop XBarrier::load_barrier_on_phantom_oop_field_preloaded(volatile narrowOop* p, oop o) {
+  ShouldNotReachHere();
+  return NULL;
+}
+
+oop XBarrier::weak_load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o) {
+  ShouldNotReachHere();
+  return NULL;
+}
+
+oop XBarrier::weak_load_barrier_on_weak_oop_field_preloaded(volatile narrowOop* p, oop o) {
+  ShouldNotReachHere();
+  return NULL;
+}
+
+oop XBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(volatile narrowOop* p, oop o) {
+  ShouldNotReachHere();
+  return NULL;
+}
+
+#ifdef ASSERT
+
+// ON_WEAK barriers should only ever be applied to j.l.r.Reference.referents.
+void XBarrier::verify_on_weak(volatile oop* referent_addr) {
+  if (referent_addr != NULL) {
+    uintptr_t base = (uintptr_t)referent_addr - java_lang_ref_Reference::referent_offset();
+    oop obj = cast_to_oop(base);
+    assert(oopDesc::is_oop(obj), "Verification failed for: ref " PTR_FORMAT " obj: " PTR_FORMAT, (uintptr_t)referent_addr, base);
+    assert(java_lang_ref_Reference::is_referent_field(obj, java_lang_ref_Reference::referent_offset()), "Sanity");
+  }
+}
+
+#endif
+
+void XLoadBarrierOopClosure::do_oop(oop* p) {
+  XBarrier::load_barrier_on_oop_field(p);
+}
+
+void XLoadBarrierOopClosure::do_oop(narrowOop* p) {
+  ShouldNotReachHere();
+}
diff --git a/src/hotspot/share/gc/x/xBarrier.hpp b/src/hotspot/share/gc/x/xBarrier.hpp
new file mode 100644
index 00000000000..e2ef210d7d2
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBarrier.hpp
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XBARRIER_HPP
+#define SHARE_GC_X_XBARRIER_HPP
+
+#include "memory/allStatic.hpp"
+#include "memory/iterator.hpp"
+#include "oops/oop.hpp"
+
+typedef bool (*XBarrierFastPath)(uintptr_t);
+typedef uintptr_t (*XBarrierSlowPath)(uintptr_t);
+
+class XBarrier : public AllStatic {
+private:
+  static const bool GCThread    = true;
+  static const bool AnyThread   = false;
+
+  static const bool Follow      = true;
+  static const bool DontFollow  = false;
+
+  static const bool Strong      = false;
+  static const bool Finalizable = true;
+
+  static const bool Publish     = true;
+  static const bool Overflow    = false;
+
+  template <XBarrierFastPath fast_path> static void self_heal(volatile oop* p, uintptr_t addr, uintptr_t heal_addr);
+
+  template <XBarrierFastPath fast_path, XBarrierSlowPath slow_path> static oop barrier(volatile oop* p, oop o);
+  template <XBarrierFastPath fast_path, XBarrierSlowPath slow_path> static oop weak_barrier(volatile oop* p, oop o);
+  template <XBarrierFastPath fast_path, XBarrierSlowPath slow_path> static void root_barrier(oop* p, oop o);
+
+  static bool is_good_or_null_fast_path(uintptr_t addr);
+  static bool is_weak_good_or_null_fast_path(uintptr_t addr);
+  static bool is_marked_or_null_fast_path(uintptr_t addr);
+
+  static bool during_mark();
+  static bool during_relocate();
+  template <bool finalizable> static bool should_mark_through(uintptr_t addr);
+  template <bool gc_thread, bool follow, bool finalizable, bool publish> static uintptr_t mark(uintptr_t addr);
+  static uintptr_t remap(uintptr_t addr);
+  static uintptr_t relocate(uintptr_t addr);
+  static uintptr_t relocate_or_mark(uintptr_t addr);
+  static uintptr_t relocate_or_mark_no_follow(uintptr_t addr);
+  static uintptr_t relocate_or_remap(uintptr_t addr);
+
+  static uintptr_t load_barrier_on_oop_slow_path(uintptr_t addr);
+  static uintptr_t load_barrier_on_invisible_root_oop_slow_path(uintptr_t addr);
+
+  static uintptr_t weak_load_barrier_on_oop_slow_path(uintptr_t addr);
+  static uintptr_t weak_load_barrier_on_weak_oop_slow_path(uintptr_t addr);
+  static uintptr_t weak_load_barrier_on_phantom_oop_slow_path(uintptr_t addr);
+
+  static uintptr_t keep_alive_barrier_on_oop_slow_path(uintptr_t addr);
+  static uintptr_t keep_alive_barrier_on_weak_oop_slow_path(uintptr_t addr);
+  static uintptr_t keep_alive_barrier_on_phantom_oop_slow_path(uintptr_t addr);
+
+  static uintptr_t mark_barrier_on_oop_slow_path(uintptr_t addr);
+  static uintptr_t mark_barrier_on_finalizable_oop_slow_path(uintptr_t addr);
+
+  static void verify_on_weak(volatile oop* referent_addr) NOT_DEBUG_RETURN;
+
+public:
+  // Load barrier
+  static  oop load_barrier_on_oop(oop o);
+  static  oop load_barrier_on_oop_field(volatile oop* p);
+  static  oop load_barrier_on_oop_field_preloaded(volatile oop* p, oop o);
+  static void load_barrier_on_oop_array(volatile oop* p, size_t length);
+  static void load_barrier_on_oop_fields(oop o);
+  static  oop load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o);
+  static  oop load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o);
+  static void load_barrier_on_root_oop_field(oop* p);
+  static void load_barrier_on_invisible_root_oop_field(oop* p);
+
+  // Weak load barrier
+  static oop weak_load_barrier_on_oop_field(volatile oop* p);
+  static oop weak_load_barrier_on_oop_field_preloaded(volatile oop* p, oop o);
+  static oop weak_load_barrier_on_weak_oop(oop o);
+  static oop weak_load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o);
+  static oop weak_load_barrier_on_phantom_oop(oop o);
+  static oop weak_load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o);
+
+  // Is alive barrier
+  static bool is_alive_barrier_on_weak_oop(oop o);
+  static bool is_alive_barrier_on_phantom_oop(oop o);
+
+  // Keep alive barrier
+  static void keep_alive_barrier_on_oop(oop o);
+  static void keep_alive_barrier_on_weak_oop_field(volatile oop* p);
+  static void keep_alive_barrier_on_phantom_oop_field(volatile oop* p);
+  static void keep_alive_barrier_on_phantom_root_oop_field(oop* p);
+
+  // Mark barrier
+  static void mark_barrier_on_oop_field(volatile oop* p, bool finalizable);
+  static void mark_barrier_on_oop_array(volatile oop* p, size_t length, bool finalizable);
+
+  // Narrow oop variants, never used.
+  static oop  load_barrier_on_oop_field(volatile narrowOop* p);
+  static oop  load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o);
+  static void load_barrier_on_oop_array(volatile narrowOop* p, size_t length);
+  static oop  load_barrier_on_weak_oop_field_preloaded(volatile narrowOop* p, oop o);
+  static oop  load_barrier_on_phantom_oop_field_preloaded(volatile narrowOop* p, oop o);
+  static oop  weak_load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o);
+  static oop  weak_load_barrier_on_weak_oop_field_preloaded(volatile narrowOop* p, oop o);
+  static oop  weak_load_barrier_on_phantom_oop_field_preloaded(volatile narrowOop* p, oop o);
+};
+
+class XLoadBarrierOopClosure : public BasicOopIterateClosure {
+public:
+  virtual void do_oop(oop* p);
+  virtual void do_oop(narrowOop* p);
+};
+
+#endif // SHARE_GC_X_XBARRIER_HPP
diff --git a/src/hotspot/share/gc/x/xBarrier.inline.hpp b/src/hotspot/share/gc/x/xBarrier.inline.hpp
new file mode 100644
index 00000000000..70288b5daac
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBarrier.inline.hpp
@@ -0,0 +1,394 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XBARRIER_INLINE_HPP
+#define SHARE_GC_X_XBARRIER_INLINE_HPP
+
+#include "gc/x/xBarrier.hpp"
+
+#include "code/codeCache.hpp"
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xOop.inline.hpp"
+#include "gc/x/xResurrection.inline.hpp"
+#include "oops/oop.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/continuation.hpp"
+
+// A self heal must always "upgrade" the address metadata bits in
+// accordance with the metadata bits state machine, which has the
+// valid state transitions as described below (where N is the GC
+// cycle).
+//
+// Note the subtleness of overlapping GC cycles. Specifically that
+// oops are colored Remapped(N) starting at relocation N and ending
+// at marking N + 1.
+//
+//              +--- Mark Start
+//              | +--- Mark End
+//              | | +--- Relocate Start
+//              | | | +--- Relocate End
+//              | | | |
+// Marked       |---N---|--N+1--|--N+2--|----
+// Finalizable  |---N---|--N+1--|--N+2--|----
+// Remapped     ----|---N---|--N+1--|--N+2--|
+//
+// VALID STATE TRANSITIONS
+//
+//   Marked(N)           -> Remapped(N)
+//                       -> Marked(N + 1)
+//                       -> Finalizable(N + 1)
+//
+//   Finalizable(N)      -> Marked(N)
+//                       -> Remapped(N)
+//                       -> Marked(N + 1)
+//                       -> Finalizable(N + 1)
+//
+//   Remapped(N)         -> Marked(N + 1)
+//                       -> Finalizable(N + 1)
+//
+// PHASE VIEW
+//
+// XPhaseMark
+//   Load & Mark
+//     Marked(N)         <- Marked(N - 1)
+//                       <- Finalizable(N - 1)
+//                       <- Remapped(N - 1)
+//                       <- Finalizable(N)
+//
+//   Mark(Finalizable)
+//     Finalizable(N)    <- Marked(N - 1)
+//                       <- Finalizable(N - 1)
+//                       <- Remapped(N - 1)
+//
+//   Load(AS_NO_KEEPALIVE)
+//     Remapped(N - 1)   <- Marked(N - 1)
+//                       <- Finalizable(N - 1)
+//
+// XPhaseMarkCompleted (Resurrection blocked)
+//   Load & Load(ON_WEAK/PHANTOM_OOP_REF | AS_NO_KEEPALIVE) & KeepAlive
+//     Marked(N)         <- Marked(N - 1)
+//                       <- Finalizable(N - 1)
+//                       <- Remapped(N - 1)
+//                       <- Finalizable(N)
+//
+//   Load(ON_STRONG_OOP_REF | AS_NO_KEEPALIVE)
+//     Remapped(N - 1)   <- Marked(N - 1)
+//                       <- Finalizable(N - 1)
+//
+// XPhaseMarkCompleted (Resurrection unblocked)
+//   Load
+//     Marked(N)         <- Finalizable(N)
+//
+// XPhaseRelocate
+//   Load & Load(AS_NO_KEEPALIVE)
+//     Remapped(N)       <- Marked(N)
+//                       <- Finalizable(N)
+
+template <XBarrierFastPath fast_path>
+inline void XBarrier::self_heal(volatile oop* p, uintptr_t addr, uintptr_t heal_addr) {
+  if (heal_addr == 0) {
+    // Never heal with null since it interacts badly with reference processing.
+    // A mutator clearing an oop would be similar to calling Reference.clear(),
+    // which would make the reference non-discoverable or silently dropped
+    // by the reference processor.
+    return;
+  }
+
+  assert(!fast_path(addr), "Invalid self heal");
+  assert(fast_path(heal_addr), "Invalid self heal");
+
+  for (;;) {
+    // Heal
+    const uintptr_t prev_addr = Atomic::cmpxchg((volatile uintptr_t*)p, addr, heal_addr, memory_order_relaxed);
+    if (prev_addr == addr) {
+      // Success
+      return;
+    }
+
+    if (fast_path(prev_addr)) {
+      // Must not self heal
+      return;
+    }
+
+    // The oop location was healed by another barrier, but still needs upgrading.
+    // Re-apply healing to make sure the oop is not left with weaker (remapped or
+    // finalizable) metadata bits than what this barrier tried to apply.
+    assert(XAddress::offset(prev_addr) == XAddress::offset(heal_addr), "Invalid offset");
+    addr = prev_addr;
+  }
+}
+
+template <XBarrierFastPath fast_path, XBarrierSlowPath slow_path>
+inline oop XBarrier::barrier(volatile oop* p, oop o) {
+  const uintptr_t addr = XOop::to_address(o);
+
+  // Fast path
+  if (fast_path(addr)) {
+    return XOop::from_address(addr);
+  }
+
+  // Slow path
+  const uintptr_t good_addr = slow_path(addr);
+
+  if (p != NULL) {
+    self_heal<fast_path>(p, addr, good_addr);
+  }
+
+  return XOop::from_address(good_addr);
+}
+
+template <XBarrierFastPath fast_path, XBarrierSlowPath slow_path>
+inline oop XBarrier::weak_barrier(volatile oop* p, oop o) {
+  const uintptr_t addr = XOop::to_address(o);
+
+  // Fast path
+  if (fast_path(addr)) {
+    // Return the good address instead of the weak good address
+    // to ensure that the currently active heap view is used.
+    return XOop::from_address(XAddress::good_or_null(addr));
+  }
+
+  // Slow path
+  const uintptr_t good_addr = slow_path(addr);
+
+  if (p != NULL) {
+    // The slow path returns a good/marked address or null, but we never mark
+    // oops in a weak load barrier so we always heal with the remapped address.
+    self_heal<fast_path>(p, addr, XAddress::remapped_or_null(good_addr));
+  }
+
+  return XOop::from_address(good_addr);
+}
+
+template <XBarrierFastPath fast_path, XBarrierSlowPath slow_path>
+inline void XBarrier::root_barrier(oop* p, oop o) {
+  const uintptr_t addr = XOop::to_address(o);
+
+  // Fast path
+  if (fast_path(addr)) {
+    return;
+  }
+
+  // Slow path
+  const uintptr_t good_addr = slow_path(addr);
+
+  // Non-atomic healing helps speed up root scanning. This is safe to do
+  // since we are always healing roots in a safepoint, or under a lock,
+  // which ensures we are never racing with mutators modifying roots while
+  // we are healing them. It's also safe in case multiple GC threads try
+  // to heal the same root if it is aligned, since they would always heal
+  // the root in the same way and it does not matter in which order it
+  // happens. For misaligned oops, there needs to be mutual exclusion.
+  *p = XOop::from_address(good_addr);
+}
+
+inline bool XBarrier::is_good_or_null_fast_path(uintptr_t addr) {
+  return XAddress::is_good_or_null(addr);
+}
+
+inline bool XBarrier::is_weak_good_or_null_fast_path(uintptr_t addr) {
+  return XAddress::is_weak_good_or_null(addr);
+}
+
+inline bool XBarrier::is_marked_or_null_fast_path(uintptr_t addr) {
+  return XAddress::is_marked_or_null(addr);
+}
+
+inline bool XBarrier::during_mark() {
+  return XGlobalPhase == XPhaseMark;
+}
+
+inline bool XBarrier::during_relocate() {
+  return XGlobalPhase == XPhaseRelocate;
+}
+
+//
+// Load barrier
+//
+inline oop XBarrier::load_barrier_on_oop(oop o) {
+  return load_barrier_on_oop_field_preloaded((oop*)NULL, o);
+}
+
+inline oop XBarrier::load_barrier_on_oop_field(volatile oop* p) {
+  const oop o = Atomic::load(p);
+  return load_barrier_on_oop_field_preloaded(p, o);
+}
+
+inline oop XBarrier::load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) {
+  return barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o);
+}
+
+inline void XBarrier::load_barrier_on_oop_array(volatile oop* p, size_t length) {
+  for (volatile const oop* const end = p + length; p < end; p++) {
+    load_barrier_on_oop_field(p);
+  }
+}
+
+inline oop XBarrier::load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) {
+  verify_on_weak(p);
+
+  if (XResurrection::is_blocked()) {
+    return barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o);
+  }
+
+  return load_barrier_on_oop_field_preloaded(p, o);
+}
+
+inline oop XBarrier::load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) {
+  if (XResurrection::is_blocked()) {
+    return barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o);
+  }
+
+  return load_barrier_on_oop_field_preloaded(p, o);
+}
+
+inline void XBarrier::load_barrier_on_root_oop_field(oop* p) {
+  const oop o = *p;
+  root_barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o);
+}
+
+inline void XBarrier::load_barrier_on_invisible_root_oop_field(oop* p) {
+  const oop o = *p;
+  root_barrier<is_good_or_null_fast_path, load_barrier_on_invisible_root_oop_slow_path>(p, o);
+}
+
+//
+// Weak load barrier
+//
+inline oop XBarrier::weak_load_barrier_on_oop_field(volatile oop* p) {
+  assert(!XResurrection::is_blocked(), "Should not be called during resurrection blocked phase");
+  const oop o = Atomic::load(p);
+  return weak_load_barrier_on_oop_field_preloaded(p, o);
+}
+
+inline oop XBarrier::weak_load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) {
+  return weak_barrier<is_weak_good_or_null_fast_path, weak_load_barrier_on_oop_slow_path>(p, o);
+}
+
+inline oop XBarrier::weak_load_barrier_on_weak_oop(oop o) {
+  return weak_load_barrier_on_weak_oop_field_preloaded((oop*)NULL, o);
+}
+
+inline oop XBarrier::weak_load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) {
+  verify_on_weak(p);
+
+  if (XResurrection::is_blocked()) {
+    return barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o);
+  }
+
+  return weak_load_barrier_on_oop_field_preloaded(p, o);
+}
+
+inline oop XBarrier::weak_load_barrier_on_phantom_oop(oop o) {
+  return weak_load_barrier_on_phantom_oop_field_preloaded((oop*)NULL, o);
+}
+
+inline oop XBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) {
+  if (XResurrection::is_blocked()) {
+    return barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o);
+  }
+
+  return weak_load_barrier_on_oop_field_preloaded(p, o);
+}
+
+//
+// Is alive barrier
+//
+inline bool XBarrier::is_alive_barrier_on_weak_oop(oop o) {
+  // Check if oop is logically non-null. This operation
+  // is only valid when resurrection is blocked.
+  assert(XResurrection::is_blocked(), "Invalid phase");
+  return weak_load_barrier_on_weak_oop(o) != NULL;
+}
+
+inline bool XBarrier::is_alive_barrier_on_phantom_oop(oop o) {
+  // Check if oop is logically non-null. This operation
+  // is only valid when resurrection is blocked.
+  assert(XResurrection::is_blocked(), "Invalid phase");
+  return weak_load_barrier_on_phantom_oop(o) != NULL;
+}
+
+//
+// Keep alive barrier
+//
+inline void XBarrier::keep_alive_barrier_on_weak_oop_field(volatile oop* p) {
+  assert(XResurrection::is_blocked(), "This operation is only valid when resurrection is blocked");
+  const oop o = Atomic::load(p);
+  barrier<is_good_or_null_fast_path, keep_alive_barrier_on_weak_oop_slow_path>(p, o);
+}
+
+inline void XBarrier::keep_alive_barrier_on_phantom_oop_field(volatile oop* p) {
+  assert(XResurrection::is_blocked(), "This operation is only valid when resurrection is blocked");
+  const oop o = Atomic::load(p);
+  barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o);
+}
+
+inline void XBarrier::keep_alive_barrier_on_phantom_root_oop_field(oop* p) {
+  // The keep alive operation is only valid when resurrection is blocked.
+  //
+  // Except with Loom, where we intentionally trigger arms nmethods after
+  // unlinking, to get a sense of what nmethods are alive. This will trigger
+  // the keep alive barriers, but the oops are healed and the slow-paths
+  // will not trigger. We have stronger checks in the slow-paths.
+  assert(XResurrection::is_blocked() || (CodeCache::contains((void*)p)),
+         "This operation is only valid when resurrection is blocked");
+  const oop o = *p;
+  root_barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o);
+}
+
+inline void XBarrier::keep_alive_barrier_on_oop(oop o) {
+  const uintptr_t addr = XOop::to_address(o);
+  assert(XAddress::is_good(addr), "Invalid address");
+
+  if (during_mark()) {
+    keep_alive_barrier_on_oop_slow_path(addr);
+  }
+}
+
+//
+// Mark barrier
+//
+inline void XBarrier::mark_barrier_on_oop_field(volatile oop* p, bool finalizable) {
+  const oop o = Atomic::load(p);
+
+  if (finalizable) {
+    barrier<is_marked_or_null_fast_path, mark_barrier_on_finalizable_oop_slow_path>(p, o);
+  } else {
+    const uintptr_t addr = XOop::to_address(o);
+    if (XAddress::is_good(addr)) {
+      // Mark through good oop
+      mark_barrier_on_oop_slow_path(addr);
+    } else {
+      // Mark through bad oop
+      barrier<is_good_or_null_fast_path, mark_barrier_on_oop_slow_path>(p, o);
+    }
+  }
+}
+
+inline void XBarrier::mark_barrier_on_oop_array(volatile oop* p, size_t length, bool finalizable) {
+  for (volatile const oop* const end = p + length; p < end; p++) {
+    mark_barrier_on_oop_field(p, finalizable);
+  }
+}
+
+#endif // SHARE_GC_X_XBARRIER_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xBarrierSet.cpp b/src/hotspot/share/gc/x/xBarrierSet.cpp
new file mode 100644
index 00000000000..cee53e8c3fa
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBarrierSet.cpp
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xBarrierSet.hpp"
+#include "gc/x/xBarrierSetAssembler.hpp"
+#include "gc/x/xBarrierSetNMethod.hpp"
+#include "gc/x/xBarrierSetStackChunk.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xHeap.inline.hpp"
+#include "gc/x/xStackWatermark.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+#include "runtime/javaThread.hpp"
+#include "utilities/macros.hpp"
+#ifdef COMPILER1
+#include "gc/x/c1/xBarrierSetC1.hpp"
+#endif
+#ifdef COMPILER2
+#include "gc/x/c2/xBarrierSetC2.hpp"
+#endif
+
+class XBarrierSetC1;
+class XBarrierSetC2;
+
+XBarrierSet::XBarrierSet() :
+    BarrierSet(make_barrier_set_assembler<XBarrierSetAssembler>(),
+               make_barrier_set_c1<XBarrierSetC1>(),
+               make_barrier_set_c2<XBarrierSetC2>(),
+               new XBarrierSetNMethod(),
+               new XBarrierSetStackChunk(),
+               BarrierSet::FakeRtti(BarrierSet::XBarrierSet)) {}
+
+XBarrierSetAssembler* XBarrierSet::assembler() {
+  BarrierSetAssembler* const bsa = BarrierSet::barrier_set()->barrier_set_assembler();
+  return reinterpret_cast<XBarrierSetAssembler*>(bsa);
+}
+
+bool XBarrierSet::barrier_needed(DecoratorSet decorators, BasicType type) {
+  assert((decorators & AS_RAW) == 0, "Unexpected decorator");
+  //assert((decorators & ON_UNKNOWN_OOP_REF) == 0, "Unexpected decorator");
+
+  if (is_reference_type(type)) {
+    assert((decorators & (IN_HEAP | IN_NATIVE)) != 0, "Where is reference?");
+    // Barrier needed even when IN_NATIVE, to allow concurrent scanning.
+    return true;
+  }
+
+  // Barrier not needed
+  return false;
+}
+
+void XBarrierSet::on_thread_create(Thread* thread) {
+  // Create thread local data
+  XThreadLocalData::create(thread);
+}
+
+void XBarrierSet::on_thread_destroy(Thread* thread) {
+  // Destroy thread local data
+  XThreadLocalData::destroy(thread);
+}
+
+void XBarrierSet::on_thread_attach(Thread* thread) {
+  // Set thread local address bad mask
+  XThreadLocalData::set_address_bad_mask(thread, XAddressBadMask);
+  if (thread->is_Java_thread()) {
+    JavaThread* const jt = JavaThread::cast(thread);
+    StackWatermark* const watermark = new XStackWatermark(jt);
+    StackWatermarkSet::add_watermark(jt, watermark);
+  }
+}
+
+void XBarrierSet::on_thread_detach(Thread* thread) {
+  // Flush and free any remaining mark stacks
+  XHeap::heap()->mark_flush_and_free(thread);
+}
+
+void XBarrierSet::print_on(outputStream* st) const {
+  st->print_cr("XBarrierSet");
+}
diff --git a/src/hotspot/share/gc/x/xBarrierSet.hpp b/src/hotspot/share/gc/x/xBarrierSet.hpp
new file mode 100644
index 00000000000..3f1eb760033
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBarrierSet.hpp
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XBARRIERSET_HPP
+#define SHARE_GC_X_XBARRIERSET_HPP
+
+#include "gc/shared/barrierSet.hpp"
+
+class XBarrierSetAssembler;
+
+class XBarrierSet : public BarrierSet {
+public:
+  XBarrierSet();
+
+  static XBarrierSetAssembler* assembler();
+  static bool barrier_needed(DecoratorSet decorators, BasicType type);
+
+  virtual void on_thread_create(Thread* thread);
+  virtual void on_thread_destroy(Thread* thread);
+  virtual void on_thread_attach(Thread* thread);
+  virtual void on_thread_detach(Thread* thread);
+
+  virtual void print_on(outputStream* st) const;
+
+  template <DecoratorSet decorators, typename BarrierSetT = XBarrierSet>
+  class AccessBarrier : public BarrierSet::AccessBarrier<decorators, BarrierSetT> {
+  private:
+    typedef BarrierSet::AccessBarrier<decorators, BarrierSetT> Raw;
+
+    template <DecoratorSet expected>
+    static void verify_decorators_present();
+
+    template <DecoratorSet expected>
+    static void verify_decorators_absent();
+
+    static oop* field_addr(oop base, ptrdiff_t offset);
+
+    template <typename T>
+    static oop load_barrier_on_oop_field_preloaded(T* addr, oop o);
+
+    template <typename T>
+    static oop load_barrier_on_unknown_oop_field_preloaded(oop base, ptrdiff_t offset, T* addr, oop o);
+
+  public:
+    //
+    // In heap
+    //
+    template <typename T>
+    static oop oop_load_in_heap(T* addr);
+    static oop oop_load_in_heap_at(oop base, ptrdiff_t offset);
+
+    template <typename T>
+    static oop oop_atomic_cmpxchg_in_heap(T* addr, oop compare_value, oop new_value);
+    static oop oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value);
+
+    template <typename T>
+    static oop oop_atomic_xchg_in_heap(T* addr, oop new_value);
+    static oop oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value);
+
+    template <typename T>
+    static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
+                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
+                                      size_t length);
+
+    static void clone_in_heap(oop src, oop dst, size_t size);
+
+    //
+    // Not in heap
+    //
+    template <typename T>
+    static oop oop_load_not_in_heap(T* addr);
+
+    template <typename T>
+    static oop oop_atomic_cmpxchg_not_in_heap(T* addr, oop compare_value, oop new_value);
+
+    template <typename T>
+    static oop oop_atomic_xchg_not_in_heap(T* addr, oop new_value);
+  };
+};
+
+template<> struct BarrierSet::GetName<XBarrierSet> {
+  static const BarrierSet::Name value = BarrierSet::XBarrierSet;
+};
+
+template<> struct BarrierSet::GetType<BarrierSet::XBarrierSet> {
+  typedef ::XBarrierSet type;
+};
+
+#endif // SHARE_GC_X_XBARRIERSET_HPP
diff --git a/src/hotspot/share/gc/x/xBarrierSet.inline.hpp b/src/hotspot/share/gc/x/xBarrierSet.inline.hpp
new file mode 100644
index 00000000000..a8ec7304e28
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBarrierSet.inline.hpp
@@ -0,0 +1,242 @@
+/*
+ * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XBARRIERSET_INLINE_HPP
+#define SHARE_GC_X_XBARRIERSET_INLINE_HPP
+
+#include "gc/x/xBarrierSet.hpp"
+
+#include "gc/shared/accessBarrierSupport.inline.hpp"
+#include "gc/x/xBarrier.inline.hpp"
+#include "utilities/debug.hpp"
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <DecoratorSet expected>
+inline void XBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_present() {
+  if ((decorators & expected) == 0) {
+    fatal("Using unsupported access decorators");
+  }
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <DecoratorSet expected>
+inline void XBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorators_absent() {
+  if ((decorators & expected) != 0) {
+    fatal("Using unsupported access decorators");
+  }
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline oop* XBarrierSet::AccessBarrier<decorators, BarrierSetT>::field_addr(oop base, ptrdiff_t offset) {
+  assert(base != NULL, "Invalid base");
+  return reinterpret_cast<oop*>(reinterpret_cast<intptr_t>((void*)base) + offset);
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline oop XBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier_on_oop_field_preloaded(T* addr, oop o) {
+  verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
+
+  if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
+    if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
+      return XBarrier::weak_load_barrier_on_oop_field_preloaded(addr, o);
+    } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
+      return XBarrier::weak_load_barrier_on_weak_oop_field_preloaded(addr, o);
+    } else {
+      assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
+      return XBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(addr, o);
+    }
+  } else {
+    if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
+      return XBarrier::load_barrier_on_oop_field_preloaded(addr, o);
+    } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
+      return XBarrier::load_barrier_on_weak_oop_field_preloaded(addr, o);
+    } else {
+      assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
+      return XBarrier::load_barrier_on_phantom_oop_field_preloaded(addr, o);
+    }
+  }
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline oop XBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier_on_unknown_oop_field_preloaded(oop base, ptrdiff_t offset, T* addr, oop o) {
+  verify_decorators_present<ON_UNKNOWN_OOP_REF>();
+
+  const DecoratorSet decorators_known_strength =
+    AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
+
+  if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
+    if (decorators_known_strength & ON_STRONG_OOP_REF) {
+      return XBarrier::weak_load_barrier_on_oop_field_preloaded(addr, o);
+    } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
+      return XBarrier::weak_load_barrier_on_weak_oop_field_preloaded(addr, o);
+    } else {
+      assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
+      return XBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(addr, o);
+    }
+  } else {
+    if (decorators_known_strength & ON_STRONG_OOP_REF) {
+      return XBarrier::load_barrier_on_oop_field_preloaded(addr, o);
+    } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
+      return XBarrier::load_barrier_on_weak_oop_field_preloaded(addr, o);
+    } else {
+      assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
+      return XBarrier::load_barrier_on_phantom_oop_field_preloaded(addr, o);
+    }
+  }
+}
+
+//
+// In heap
+//
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline oop XBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) {
+  verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
+
+  const oop o = Raw::oop_load_in_heap(addr);
+  return load_barrier_on_oop_field_preloaded(addr, o);
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline oop XBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
+  oop* const addr = field_addr(base, offset);
+  const oop o = Raw::oop_load_in_heap(addr);
+
+  if (HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) {
+    return load_barrier_on_unknown_oop_field_preloaded(base, offset, addr, o);
+  }
+
+  return load_barrier_on_oop_field_preloaded(addr, o);
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline oop XBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(T* addr, oop compare_value, oop new_value) {
+  verify_decorators_present<ON_STRONG_OOP_REF>();
+  verify_decorators_absent<AS_NO_KEEPALIVE>();
+
+  XBarrier::load_barrier_on_oop_field(addr);
+  return Raw::oop_atomic_cmpxchg_in_heap(addr, compare_value, new_value);
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline oop XBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value) {
+  verify_decorators_present<ON_STRONG_OOP_REF | ON_UNKNOWN_OOP_REF>();
+  verify_decorators_absent<AS_NO_KEEPALIVE>();
+
+  // Through Unsafe.CompareAndExchangeObject()/CompareAndSetObject() we can receive
+  // calls with ON_UNKNOWN_OOP_REF set. However, we treat these as ON_STRONG_OOP_REF,
+  // with the motivation that if you're doing Unsafe operations on a Reference.referent
+  // field, then you're on your own anyway.
+  XBarrier::load_barrier_on_oop_field(field_addr(base, offset));
+  return Raw::oop_atomic_cmpxchg_in_heap_at(base, offset, compare_value, new_value);
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline oop XBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(T* addr, oop new_value) {
+  verify_decorators_present<ON_STRONG_OOP_REF>();
+  verify_decorators_absent<AS_NO_KEEPALIVE>();
+
+  const oop o = Raw::oop_atomic_xchg_in_heap(addr, new_value);
+  return XBarrier::load_barrier_on_oop(o);
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline oop XBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value) {
+  verify_decorators_present<ON_STRONG_OOP_REF>();
+  verify_decorators_absent<AS_NO_KEEPALIVE>();
+
+  const oop o = Raw::oop_atomic_xchg_in_heap_at(base, offset, new_value);
+  return XBarrier::load_barrier_on_oop(o);
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline bool XBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
+                                                                                       arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
+                                                                                       size_t length) {
+  T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
+  T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
+
+  if (!HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) {
+    // No check cast, bulk barrier and bulk copy
+    XBarrier::load_barrier_on_oop_array(src, length);
+    return Raw::oop_arraycopy_in_heap(NULL, 0, src, NULL, 0, dst, length);
+  }
+
+  // Check cast and copy each elements
+  Klass* const dst_klass = objArrayOop(dst_obj)->element_klass();
+  for (const T* const end = src + length; src < end; src++, dst++) {
+    const oop elem = XBarrier::load_barrier_on_oop_field(src);
+    if (!oopDesc::is_instanceof_or_null(elem, dst_klass)) {
+      // Check cast failed
+      return false;
+    }
+
+    // Cast is safe, since we know it's never a narrowOop
+    *(oop*)dst = elem;
+  }
+
+  return true;
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline void XBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
+  XBarrier::load_barrier_on_oop_fields(src);
+  Raw::clone_in_heap(src, dst, size);
+}
+
+//
+// Not in heap
+//
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline oop XBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) {
+  verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
+
+  const oop o = Raw::oop_load_not_in_heap(addr);
+  return load_barrier_on_oop_field_preloaded(addr, o);
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline oop XBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(T* addr, oop compare_value, oop new_value) {
+  verify_decorators_present<ON_STRONG_OOP_REF>();
+  verify_decorators_absent<AS_NO_KEEPALIVE>();
+
+  return Raw::oop_atomic_cmpxchg_not_in_heap(addr, compare_value, new_value);
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+template <typename T>
+inline oop XBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(T* addr, oop new_value) {
+  verify_decorators_present<ON_STRONG_OOP_REF>();
+  verify_decorators_absent<AS_NO_KEEPALIVE>();
+
+  return Raw::oop_atomic_xchg_not_in_heap(addr, new_value);
+}
+
+#endif // SHARE_GC_X_XBARRIERSET_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xBarrierSetAssembler.cpp b/src/hotspot/share/gc/x/xBarrierSetAssembler.cpp
new file mode 100644
index 00000000000..d00c12ed291
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBarrierSetAssembler.cpp
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xBarrierSetAssembler.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+#include "runtime/javaThread.hpp"
+
+Address XBarrierSetAssemblerBase::address_bad_mask_from_thread(Register thread) {
+  return Address(thread, XThreadLocalData::address_bad_mask_offset());
+}
+
+Address XBarrierSetAssemblerBase::address_bad_mask_from_jni_env(Register env) {
+  return Address(env, XThreadLocalData::address_bad_mask_offset() - JavaThread::jni_environment_offset());
+}
diff --git a/src/hotspot/share/gc/x/xBarrierSetAssembler.hpp b/src/hotspot/share/gc/x/xBarrierSetAssembler.hpp
new file mode 100644
index 00000000000..2f733465bfb
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBarrierSetAssembler.hpp
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XBARRIERSETASSEMBLER_HPP
+#define SHARE_GC_X_XBARRIERSETASSEMBLER_HPP
+
+#include "gc/shared/barrierSetAssembler.hpp"
+#include "utilities/macros.hpp"
+
+class XBarrierSetAssemblerBase : public BarrierSetAssembler {
+public:
+  static Address address_bad_mask_from_thread(Register thread);
+  static Address address_bad_mask_from_jni_env(Register env);
+};
+
+// Needs to be included after definition of XBarrierSetAssemblerBase
+#include CPU_HEADER(gc/x/xBarrierSetAssembler)
+
+#endif // SHARE_GC_X_XBARRIERSETASSEMBLER_HPP
diff --git a/src/hotspot/share/gc/x/xBarrierSetNMethod.cpp b/src/hotspot/share/gc/x/xBarrierSetNMethod.cpp
new file mode 100644
index 00000000000..002d6bc00c5
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBarrierSetNMethod.cpp
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "code/nmethod.hpp"
+#include "gc/x/xBarrierSetNMethod.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xLock.inline.hpp"
+#include "gc/x/xNMethod.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+#include "logging/log.hpp"
+#include "runtime/threadWXSetters.inline.hpp"
+
+bool XBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) {
+  XLocker<XReentrantLock> locker(XNMethod::lock_for_nmethod(nm));
+  log_trace(nmethod, barrier)("Entered critical zone for %p", nm);
+
+  if (!is_armed(nm)) {
+    // Some other thread got here first and healed the oops
+    // and disarmed the nmethod.
+    return true;
+  }
+
+  MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, Thread::current()));
+
+  if (nm->is_unloading()) {
+    // We don't need to take the lock when unlinking nmethods from
+    // the Method, because it is only concurrently unlinked by
+    // the entry barrier, which acquires the per nmethod lock.
+    nm->unlink_from_method();
+
+    // We can end up calling nmethods that are unloading
+    // since we clear compiled ICs lazily. Returning false
+    // will re-resovle the call and update the compiled IC.
+    return false;
+  }
+
+  // Heal oops
+  XNMethod::nmethod_oops_barrier(nm);
+
+
+  // CodeCache unloading support
+  nm->mark_as_maybe_on_stack();
+
+  // Disarm
+  disarm(nm);
+
+  return true;
+}
+
+int* XBarrierSetNMethod::disarmed_guard_value_address() const {
+  return (int*)XAddressBadMaskHighOrderBitsAddr;
+}
+
+ByteSize XBarrierSetNMethod::thread_disarmed_guard_value_offset() const {
+  return XThreadLocalData::nmethod_disarmed_offset();
+}
diff --git a/src/hotspot/share/gc/x/xBarrierSetNMethod.hpp b/src/hotspot/share/gc/x/xBarrierSetNMethod.hpp
new file mode 100644
index 00000000000..db1ee8c4e8f
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBarrierSetNMethod.hpp
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XBARRIERSETNMETHOD_HPP
+#define SHARE_GC_X_XBARRIERSETNMETHOD_HPP
+
+#include "gc/shared/barrierSetNMethod.hpp"
+#include "memory/allocation.hpp"
+
+class nmethod;
+
+class XBarrierSetNMethod : public BarrierSetNMethod {
+protected:
+  virtual bool nmethod_entry_barrier(nmethod* nm);
+
+public:
+  virtual ByteSize thread_disarmed_guard_value_offset() const;
+  virtual int* disarmed_guard_value_address() const;
+};
+
+#endif // SHARE_GC_X_XBARRIERSETNMETHOD_HPP
diff --git a/src/hotspot/share/gc/x/xBarrierSetRuntime.cpp b/src/hotspot/share/gc/x/xBarrierSetRuntime.cpp
new file mode 100644
index 00000000000..d87df24b9d8
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBarrierSetRuntime.cpp
@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xBarrier.inline.hpp"
+#include "gc/x/xBarrierSetRuntime.hpp"
+#include "oops/access.hpp"
+#include "runtime/interfaceSupport.inline.hpp"
+
+JRT_LEAF(oopDesc*, XBarrierSetRuntime::load_barrier_on_oop_field_preloaded(oopDesc* o, oop* p))
+  return XBarrier::load_barrier_on_oop_field_preloaded(p, o);
+JRT_END
+
+JRT_LEAF(oopDesc*, XBarrierSetRuntime::weak_load_barrier_on_oop_field_preloaded(oopDesc* o, oop* p))
+  return XBarrier::weak_load_barrier_on_oop_field_preloaded(p, o);
+JRT_END
+
+JRT_LEAF(oopDesc*, XBarrierSetRuntime::weak_load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p))
+  return XBarrier::weak_load_barrier_on_weak_oop_field_preloaded(p, o);
+JRT_END
+
+JRT_LEAF(oopDesc*, XBarrierSetRuntime::weak_load_barrier_on_phantom_oop_field_preloaded(oopDesc* o, oop* p))
+  return XBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(p, o);
+JRT_END
+
+JRT_LEAF(oopDesc*, XBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p))
+  return XBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
+JRT_END
+
+JRT_LEAF(oopDesc*, XBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded(oopDesc* o, oop* p))
+  return XBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
+JRT_END
+
+JRT_LEAF(void, XBarrierSetRuntime::load_barrier_on_oop_array(oop* p, size_t length))
+  XBarrier::load_barrier_on_oop_array(p, length);
+JRT_END
+
+JRT_LEAF(void, XBarrierSetRuntime::clone(oopDesc* src, oopDesc* dst, size_t size))
+  HeapAccess<>::clone(src, dst, size);
+JRT_END
+
+address XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(DecoratorSet decorators) {
+  if (decorators & ON_PHANTOM_OOP_REF) {
+    if (decorators & AS_NO_KEEPALIVE) {
+      return weak_load_barrier_on_phantom_oop_field_preloaded_addr();
+    } else {
+      return load_barrier_on_phantom_oop_field_preloaded_addr();
+    }
+  } else if (decorators & ON_WEAK_OOP_REF) {
+    if (decorators & AS_NO_KEEPALIVE) {
+      return weak_load_barrier_on_weak_oop_field_preloaded_addr();
+    } else {
+      return load_barrier_on_weak_oop_field_preloaded_addr();
+    }
+  } else {
+    if (decorators & AS_NO_KEEPALIVE) {
+      return weak_load_barrier_on_oop_field_preloaded_addr();
+    } else {
+      return load_barrier_on_oop_field_preloaded_addr();
+    }
+  }
+}
+
+address XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() {
+  return reinterpret_cast<address>(load_barrier_on_oop_field_preloaded);
+}
+
+address XBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr() {
+  return reinterpret_cast<address>(load_barrier_on_weak_oop_field_preloaded);
+}
+
+address XBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded_addr() {
+  return reinterpret_cast<address>(load_barrier_on_phantom_oop_field_preloaded);
+}
+
+address XBarrierSetRuntime::weak_load_barrier_on_oop_field_preloaded_addr() {
+  return reinterpret_cast<address>(weak_load_barrier_on_oop_field_preloaded);
+}
+
+address XBarrierSetRuntime::weak_load_barrier_on_weak_oop_field_preloaded_addr() {
+  return reinterpret_cast<address>(weak_load_barrier_on_weak_oop_field_preloaded);
+}
+
+address XBarrierSetRuntime::weak_load_barrier_on_phantom_oop_field_preloaded_addr() {
+  return reinterpret_cast<address>(weak_load_barrier_on_phantom_oop_field_preloaded);
+}
+
+address XBarrierSetRuntime::load_barrier_on_oop_array_addr() {
+  return reinterpret_cast<address>(load_barrier_on_oop_array);
+}
+
+address XBarrierSetRuntime::clone_addr() {
+  return reinterpret_cast<address>(clone);
+}
diff --git a/src/hotspot/share/gc/x/xBarrierSetRuntime.hpp b/src/hotspot/share/gc/x/xBarrierSetRuntime.hpp
new file mode 100644
index 00000000000..6302f1ce36d
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBarrierSetRuntime.hpp
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XBARRIERSETRUNTIME_HPP
+#define SHARE_GC_X_XBARRIERSETRUNTIME_HPP
+
+#include "memory/allStatic.hpp"
+#include "oops/accessDecorators.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class oopDesc;
+
+class XBarrierSetRuntime : public AllStatic {
+private:
+  static oopDesc* load_barrier_on_oop_field_preloaded(oopDesc* o, oop* p);
+  static oopDesc* load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p);
+  static oopDesc* load_barrier_on_phantom_oop_field_preloaded(oopDesc* o, oop* p);
+  static oopDesc* weak_load_barrier_on_oop_field_preloaded(oopDesc* o, oop* p);
+  static oopDesc* weak_load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p);
+  static oopDesc* weak_load_barrier_on_phantom_oop_field_preloaded(oopDesc* o, oop* p);
+  static void load_barrier_on_oop_array(oop* p, size_t length);
+  static void clone(oopDesc* src, oopDesc* dst, size_t size);
+
+public:
+  static address load_barrier_on_oop_field_preloaded_addr(DecoratorSet decorators);
+  static address load_barrier_on_oop_field_preloaded_addr();
+  static address load_barrier_on_weak_oop_field_preloaded_addr();
+  static address load_barrier_on_phantom_oop_field_preloaded_addr();
+  static address weak_load_barrier_on_oop_field_preloaded_addr();
+  static address weak_load_barrier_on_weak_oop_field_preloaded_addr();
+  static address weak_load_barrier_on_phantom_oop_field_preloaded_addr();
+  static address load_barrier_on_oop_array_addr();
+  static address clone_addr();
+};
+
+#endif // SHARE_GC_X_XBARRIERSETRUNTIME_HPP
diff --git a/src/hotspot/share/gc/x/xBarrierSetStackChunk.cpp b/src/hotspot/share/gc/x/xBarrierSetStackChunk.cpp
new file mode 100644
index 00000000000..37e36597c1d
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBarrierSetStackChunk.cpp
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xBarrier.inline.hpp"
+#include "gc/x/xBarrierSetStackChunk.hpp"
+#include "runtime/atomic.hpp"
+#include "utilities/debug.hpp"
+
+void XBarrierSetStackChunk::encode_gc_mode(stackChunkOop chunk, OopIterator* iterator) {
+  // Do nothing
+}
+
+void XBarrierSetStackChunk::decode_gc_mode(stackChunkOop chunk, OopIterator* iterator) {
+  // Do nothing
+}
+
+oop XBarrierSetStackChunk::load_oop(stackChunkOop chunk, oop* addr) {
+  oop obj = Atomic::load(addr);
+  return XBarrier::load_barrier_on_oop_field_preloaded((volatile oop*)NULL, obj);
+}
+
+oop XBarrierSetStackChunk::load_oop(stackChunkOop chunk, narrowOop* addr) {
+  ShouldNotReachHere();
+  return NULL;
+}
diff --git a/src/hotspot/share/gc/x/xBarrierSetStackChunk.hpp b/src/hotspot/share/gc/x/xBarrierSetStackChunk.hpp
new file mode 100644
index 00000000000..36180db7b8c
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBarrierSetStackChunk.hpp
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#ifndef SHARE_GC_X_XBARRIERSETSTACKCHUNK_HPP
+#define SHARE_GC_X_XBARRIERSETSTACKCHUNK_HPP
+
+#include "gc/shared/barrierSetStackChunk.hpp"
+#include "memory/iterator.hpp"
+#include "oops/oopsHierarchy.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class OopClosure;
+
+class XBarrierSetStackChunk : public BarrierSetStackChunk {
+public:
+  virtual void encode_gc_mode(stackChunkOop chunk, OopIterator* iterator) override;
+  virtual void decode_gc_mode(stackChunkOop chunk, OopIterator* iterator) override;
+
+  virtual oop load_oop(stackChunkOop chunk, oop* addr) override;
+  virtual oop load_oop(stackChunkOop chunk, narrowOop* addr) override;
+};
+
+#endif // SHARE_GC_X_XBARRIERSETSTACKCHUNK_HPP
diff --git a/src/hotspot/share/gc/x/xBitField.hpp b/src/hotspot/share/gc/x/xBitField.hpp
new file mode 100644
index 00000000000..f11d7cf7ef7
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBitField.hpp
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XBITFIELD_HPP
+#define SHARE_GC_X_XBITFIELD_HPP
+
+#include "memory/allStatic.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+//
+//  Example
+//  -------
+//
+//  typedef XBitField<uint64_t, uint8_t,  0,  2, 3> field_word_aligned_size;
+//  typedef XBitField<uint64_t, uint32_t, 2, 30>    field_length;
+//
+//
+//   6                                 3 3
+//   3                                 2 1                               2 10
+//  +-----------------------------------+---------------------------------+--+
+//  |11111111 11111111 11111111 11111111|11111111 11111111 11111111 111111|11|
+//  +-----------------------------------+---------------------------------+--+
+//  |                                   |                                 |
+//  |       31-2 field_length (30-bits) *                                 |
+//  |                                                                     |
+//  |                                1-0 field_word_aligned_size (2-bits) *
+//  |
+//  * 63-32 Unused (32-bits)
+//
+//
+//  field_word_aligned_size::encode(16) = 2
+//  field_length::encode(2342) = 9368
+//
+//  field_word_aligned_size::decode(9368 | 2) = 16
+//  field_length::decode(9368 | 2) = 2342
+//
+
+template <typename ContainerType, typename ValueType, int FieldShift, int FieldBits, int ValueShift = 0>
+class XBitField : public AllStatic {
+private:
+  static const int ContainerBits = sizeof(ContainerType) * BitsPerByte;
+
+  static_assert(FieldBits < ContainerBits, "Field too large");
+  static_assert(FieldShift + FieldBits <= ContainerBits, "Field too large");
+  static_assert(ValueShift + FieldBits <= ContainerBits, "Field too large");
+
+  static const ContainerType FieldMask = (((ContainerType)1 << FieldBits) - 1);
+
+public:
+  static ValueType decode(ContainerType container) {
+    return (ValueType)(((container >> FieldShift) & FieldMask) << ValueShift);
+  }
+
+  static ContainerType encode(ValueType value) {
+    assert(((ContainerType)value & (FieldMask << ValueShift)) == (ContainerType)value, "Invalid value");
+    return ((ContainerType)value >> ValueShift) << FieldShift;
+  }
+};
+
+#endif // SHARE_GC_X_XBITFIELD_HPP
diff --git a/src/hotspot/share/gc/x/xBitMap.hpp b/src/hotspot/share/gc/x/xBitMap.hpp
new file mode 100644
index 00000000000..c96f63b4c89
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBitMap.hpp
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XBITMAP_HPP
+#define SHARE_GC_X_XBITMAP_HPP
+
+#include "utilities/bitMap.hpp"
+
+class XBitMap : public CHeapBitMap {
+private:
+  static bm_word_t bit_mask_pair(idx_t bit);
+
+  bool par_set_bit_pair_finalizable(idx_t bit, bool& inc_live);
+  bool par_set_bit_pair_strong(idx_t bit, bool& inc_live);
+
+public:
+  XBitMap(idx_t size_in_bits);
+
+  bool par_set_bit_pair(idx_t bit, bool finalizable, bool& inc_live);
+};
+
+#endif // SHARE_GC_X_XBITMAP_HPP
diff --git a/src/hotspot/share/gc/x/xBitMap.inline.hpp b/src/hotspot/share/gc/x/xBitMap.inline.hpp
new file mode 100644
index 00000000000..e35f59eeb88
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBitMap.inline.hpp
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XBITMAP_INLINE_HPP
+#define SHARE_GC_X_XBITMAP_INLINE_HPP
+
+#include "gc/x/xBitMap.hpp"
+
+#include "runtime/atomic.hpp"
+#include "utilities/bitMap.inline.hpp"
+#include "utilities/debug.hpp"
+
+inline XBitMap::XBitMap(idx_t size_in_bits) :
+    CHeapBitMap(size_in_bits, mtGC, false /* clear */) {}
+
+inline BitMap::bm_word_t XBitMap::bit_mask_pair(idx_t bit) {
+  assert(bit_in_word(bit) < BitsPerWord - 1, "Invalid bit index");
+  return (bm_word_t)3 << bit_in_word(bit);
+}
+
+inline bool XBitMap::par_set_bit_pair_finalizable(idx_t bit, bool& inc_live) {
+  inc_live = par_set_bit(bit);
+  return inc_live;
+}
+
+inline bool XBitMap::par_set_bit_pair_strong(idx_t bit, bool& inc_live) {
+  verify_index(bit);
+  volatile bm_word_t* const addr = word_addr(bit);
+  const bm_word_t pair_mask = bit_mask_pair(bit);
+  bm_word_t old_val = *addr;
+
+  do {
+    const bm_word_t new_val = old_val | pair_mask;
+    if (new_val == old_val) {
+      // Someone else beat us to it
+      inc_live = false;
+      return false;
+    }
+    const bm_word_t cur_val = Atomic::cmpxchg(addr, old_val, new_val);
+    if (cur_val == old_val) {
+      // Success
+      const bm_word_t marked_mask = bit_mask(bit);
+      inc_live = !(old_val & marked_mask);
+      return true;
+    }
+
+    // The value changed, retry
+    old_val = cur_val;
+  } while (true);
+}
+
+inline bool XBitMap::par_set_bit_pair(idx_t bit, bool finalizable, bool& inc_live) {
+  if (finalizable) {
+    return par_set_bit_pair_finalizable(bit, inc_live);
+  } else {
+    return par_set_bit_pair_strong(bit, inc_live);
+  }
+}
+
+#endif // SHARE_GC_X_XBITMAP_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xBreakpoint.cpp b/src/hotspot/share/gc/x/xBreakpoint.cpp
new file mode 100644
index 00000000000..e053ceaedb9
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBreakpoint.cpp
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/concurrentGCBreakpoints.hpp"
+#include "gc/x/xBreakpoint.hpp"
+#include "runtime/mutexLocker.hpp"
+#include "utilities/debug.hpp"
+
+bool XBreakpoint::_start_gc = false;
+
+void XBreakpoint::start_gc() {
+  MonitorLocker ml(ConcurrentGCBreakpoints::monitor());
+  assert(ConcurrentGCBreakpoints::is_controlled(), "Invalid state");
+  assert(!_start_gc, "Invalid state");
+  _start_gc = true;
+  ml.notify_all();
+}
+
+void XBreakpoint::at_before_gc() {
+  MonitorLocker ml(ConcurrentGCBreakpoints::monitor(), Mutex::_no_safepoint_check_flag);
+  while (ConcurrentGCBreakpoints::is_controlled() && !_start_gc) {
+    ml.wait();
+  }
+  _start_gc = false;
+  ConcurrentGCBreakpoints::notify_idle_to_active();
+}
+
+void XBreakpoint::at_after_gc() {
+  ConcurrentGCBreakpoints::notify_active_to_idle();
+}
+
+void XBreakpoint::at_after_marking_started() {
+  ConcurrentGCBreakpoints::at("AFTER MARKING STARTED");
+}
+
+void XBreakpoint::at_before_marking_completed() {
+  ConcurrentGCBreakpoints::at("BEFORE MARKING COMPLETED");
+}
+
+void XBreakpoint::at_after_reference_processing_started() {
+  ConcurrentGCBreakpoints::at("AFTER CONCURRENT REFERENCE PROCESSING STARTED");
+}
diff --git a/src/hotspot/share/gc/x/xBreakpoint.hpp b/src/hotspot/share/gc/x/xBreakpoint.hpp
new file mode 100644
index 00000000000..0c0b9d3c90f
--- /dev/null
+++ b/src/hotspot/share/gc/x/xBreakpoint.hpp
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XBREAKPOINT_HPP
+#define SHARE_GC_X_XBREAKPOINT_HPP
+
+#include "memory/allStatic.hpp"
+
+class XBreakpoint : public AllStatic {
+private:
+  static bool _start_gc;
+
+public:
+  static void start_gc();
+
+  static void at_before_gc();
+  static void at_after_gc();
+  static void at_after_marking_started();
+  static void at_before_marking_completed();
+  static void at_after_reference_processing_started();
+};
+
+#endif // SHARE_GC_X_XBREAKPOINT_HPP
diff --git a/src/hotspot/share/gc/x/xCPU.cpp b/src/hotspot/share/gc/x/xCPU.cpp
new file mode 100644
index 00000000000..d212739a305
--- /dev/null
+++ b/src/hotspot/share/gc/x/xCPU.cpp
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/x/xCPU.inline.hpp"
+#include "memory/padded.inline.hpp"
+#include "runtime/javaThread.hpp"
+#include "runtime/os.hpp"
+#include "utilities/debug.hpp"
+
+#define XCPU_UNKNOWN_AFFINITY ((Thread*)-1)
+#define XCPU_UNKNOWN_SELF     ((Thread*)-2)
+
+PaddedEnd<XCPU::XCPUAffinity>* XCPU::_affinity = NULL;
+THREAD_LOCAL Thread*           XCPU::_self     = XCPU_UNKNOWN_SELF;
+THREAD_LOCAL uint32_t          XCPU::_cpu      = 0;
+
+void XCPU::initialize() {
+  assert(_affinity == NULL, "Already initialized");
+  const uint32_t ncpus = count();
+
+  _affinity = PaddedArray<XCPUAffinity, mtGC>::create_unfreeable(ncpus);
+
+  for (uint32_t i = 0; i < ncpus; i++) {
+    _affinity[i]._thread = XCPU_UNKNOWN_AFFINITY;
+  }
+
+  log_info_p(gc, init)("CPUs: %u total, %u available",
+                       os::processor_count(),
+                       os::initial_active_processor_count());
+}
+
+uint32_t XCPU::id_slow() {
+  // Set current thread
+  if (_self == XCPU_UNKNOWN_SELF) {
+    _self = Thread::current();
+  }
+
+  // Set current CPU
+  _cpu = os::processor_id();
+
+  // Update affinity table
+  _affinity[_cpu]._thread = _self;
+
+  return _cpu;
+}
diff --git a/src/hotspot/share/gc/x/xCPU.hpp b/src/hotspot/share/gc/x/xCPU.hpp
new file mode 100644
index 00000000000..fd931956c4b
--- /dev/null
+++ b/src/hotspot/share/gc/x/xCPU.hpp
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XCPU_HPP
+#define SHARE_GC_X_XCPU_HPP
+
+#include "memory/allStatic.hpp"
+#include "memory/padded.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class Thread;
+
+class XCPU : public AllStatic {
+private:
+  struct XCPUAffinity {
+    Thread* _thread;
+  };
+
+  static PaddedEnd<XCPUAffinity>* _affinity;
+  static THREAD_LOCAL Thread*     _self;
+  static THREAD_LOCAL uint32_t    _cpu;
+
+  static uint32_t id_slow();
+
+public:
+  static void initialize();
+
+  static uint32_t count();
+  static uint32_t id();
+};
+
+#endif // SHARE_GC_X_XCPU_HPP
diff --git a/src/hotspot/share/gc/x/xCPU.inline.hpp b/src/hotspot/share/gc/x/xCPU.inline.hpp
new file mode 100644
index 00000000000..ce1f4ec65c9
--- /dev/null
+++ b/src/hotspot/share/gc/x/xCPU.inline.hpp
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XCPU_INLINE_HPP
+#define SHARE_GC_X_XCPU_INLINE_HPP
+
+#include "gc/x/xCPU.hpp"
+
+#include "runtime/os.hpp"
+#include "utilities/debug.hpp"
+
+inline uint32_t XCPU::count() {
+  return os::processor_count();
+}
+
+inline uint32_t XCPU::id() {
+  assert(_affinity != NULL, "Not initialized");
+
+  // Fast path
+  if (_affinity[_cpu]._thread == _self) {
+    return _cpu;
+  }
+
+  // Slow path
+  return id_slow();
+}
+
+#endif // SHARE_GC_X_XCPU_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xCollectedHeap.cpp b/src/hotspot/share/gc/x/xCollectedHeap.cpp
new file mode 100644
index 00000000000..935441d627a
--- /dev/null
+++ b/src/hotspot/share/gc/x/xCollectedHeap.cpp
@@ -0,0 +1,351 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "classfile/classLoaderData.hpp"
+#include "gc/shared/gcHeapSummary.hpp"
+#include "gc/shared/gcLocker.inline.hpp"
+#include "gc/shared/suspendibleThreadSet.hpp"
+#include "gc/x/xCollectedHeap.hpp"
+#include "gc/x/xDirector.hpp"
+#include "gc/x/xDriver.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xHeap.inline.hpp"
+#include "gc/x/xNMethod.hpp"
+#include "gc/x/xObjArrayAllocator.hpp"
+#include "gc/x/xOop.inline.hpp"
+#include "gc/x/xServiceability.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xUtils.inline.hpp"
+#include "memory/classLoaderMetaspace.hpp"
+#include "memory/iterator.hpp"
+#include "memory/metaspaceCriticalAllocation.hpp"
+#include "memory/universe.hpp"
+#include "oops/stackChunkOop.hpp"
+#include "runtime/continuationJavaClasses.hpp"
+#include "utilities/align.hpp"
+
+XCollectedHeap* XCollectedHeap::heap() {
+  return named_heap<XCollectedHeap>(CollectedHeap::Z);
+}
+
+XCollectedHeap::XCollectedHeap() :
+    _soft_ref_policy(),
+    _barrier_set(),
+    _initialize(&_barrier_set),
+    _heap(),
+    _driver(new XDriver()),
+    _director(new XDirector(_driver)),
+    _stat(new XStat()),
+    _runtime_workers() {}
+
+CollectedHeap::Name XCollectedHeap::kind() const {
+  return CollectedHeap::Z;
+}
+
+const char* XCollectedHeap::name() const {
+  return XName;
+}
+
+jint XCollectedHeap::initialize() {
+  if (!_heap.is_initialized()) {
+    return JNI_ENOMEM;
+  }
+
+  Universe::calculate_verify_data((HeapWord*)0, (HeapWord*)UINTPTR_MAX);
+
+  return JNI_OK;
+}
+
+void XCollectedHeap::initialize_serviceability() {
+  _heap.serviceability_initialize();
+}
+
+class XStopConcurrentGCThreadClosure : public ThreadClosure {
+public:
+  virtual void do_thread(Thread* thread) {
+    if (thread->is_ConcurrentGC_thread()) {
+      ConcurrentGCThread::cast(thread)->stop();
+    }
+  }
+};
+
+void XCollectedHeap::stop() {
+  XStopConcurrentGCThreadClosure cl;
+  gc_threads_do(&cl);
+}
+
+SoftRefPolicy* XCollectedHeap::soft_ref_policy() {
+  return &_soft_ref_policy;
+}
+
+size_t XCollectedHeap::max_capacity() const {
+  return _heap.max_capacity();
+}
+
+size_t XCollectedHeap::capacity() const {
+  return _heap.capacity();
+}
+
+size_t XCollectedHeap::used() const {
+  return _heap.used();
+}
+
+size_t XCollectedHeap::unused() const {
+  return _heap.unused();
+}
+
+bool XCollectedHeap::is_maximal_no_gc() const {
+  // Not supported
+  ShouldNotReachHere();
+  return false;
+}
+
+bool XCollectedHeap::is_in(const void* p) const {
+  return _heap.is_in((uintptr_t)p);
+}
+
+bool XCollectedHeap::requires_barriers(stackChunkOop obj) const {
+  uintptr_t* cont_addr = obj->field_addr<uintptr_t>(jdk_internal_vm_StackChunk::cont_offset());
+
+  if (!_heap.is_allocating(cast_from_oop<uintptr_t>(obj))) {
+    // An object that isn't allocating, is visible from GC tracing. Such
+    // stack chunks require barriers.
+    return true;
+  }
+
+  if (!XAddress::is_good_or_null(*cont_addr)) {
+    // If a chunk is allocated after a GC started, but before relocate start
+    // we can have an allocating chunk that isn't deeply good. That means that
+    // the contained oops might be bad and require GC barriers.
+    return true;
+  }
+
+  // The chunk is allocating and its pointers are good. This chunk needs no
+  // GC barriers
+  return false;
+}
+
+HeapWord* XCollectedHeap::allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) {
+  const size_t size_in_bytes = XUtils::words_to_bytes(align_object_size(requested_size));
+  const uintptr_t addr = _heap.alloc_tlab(size_in_bytes);
+
+  if (addr != 0) {
+    *actual_size = requested_size;
+  }
+
+  return (HeapWord*)addr;
+}
+
+oop XCollectedHeap::array_allocate(Klass* klass, size_t size, int length, bool do_zero, TRAPS) {
+  XObjArrayAllocator allocator(klass, size, length, do_zero, THREAD);
+  return allocator.allocate();
+}
+
+HeapWord* XCollectedHeap::mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) {
+  const size_t size_in_bytes = XUtils::words_to_bytes(align_object_size(size));
+  return (HeapWord*)_heap.alloc_object(size_in_bytes);
+}
+
+MetaWord* XCollectedHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
+                                                             size_t size,
+                                                             Metaspace::MetadataType mdtype) {
+  // Start asynchronous GC
+  collect(GCCause::_metadata_GC_threshold);
+
+  // Expand and retry allocation
+  MetaWord* const result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
+  if (result != NULL) {
+    return result;
+  }
+
+  // As a last resort, try a critical allocation, riding on a synchronous full GC
+  return MetaspaceCriticalAllocation::allocate(loader_data, size, mdtype);
+}
+
+void XCollectedHeap::collect(GCCause::Cause cause) {
+  _driver->collect(cause);
+}
+
+void XCollectedHeap::collect_as_vm_thread(GCCause::Cause cause) {
+  // These collection requests are ignored since ZGC can't run a synchronous
+  // GC cycle from within the VM thread. This is considered benign, since the
+  // only GC causes coming in here should be heap dumper and heap inspector.
+  // If the heap dumper or heap inspector explicitly requests a gc and the
+  // caller is not the VM thread a synchronous GC cycle is performed from the
+  // caller thread in the prologue.
+  assert(Thread::current()->is_VM_thread(), "Should be the VM thread");
+  guarantee(cause == GCCause::_heap_dump ||
+            cause == GCCause::_heap_inspection, "Invalid cause");
+}
+
+void XCollectedHeap::do_full_collection(bool clear_all_soft_refs) {
+  // Not supported
+  ShouldNotReachHere();
+}
+
+size_t XCollectedHeap::tlab_capacity(Thread* ignored) const {
+  return _heap.tlab_capacity();
+}
+
+size_t XCollectedHeap::tlab_used(Thread* ignored) const {
+  return _heap.tlab_used();
+}
+
+size_t XCollectedHeap::max_tlab_size() const {
+  return _heap.max_tlab_size();
+}
+
+size_t XCollectedHeap::unsafe_max_tlab_alloc(Thread* ignored) const {
+  return _heap.unsafe_max_tlab_alloc();
+}
+
+bool XCollectedHeap::uses_stack_watermark_barrier() const {
+  return true;
+}
+
+MemoryUsage XCollectedHeap::memory_usage() {
+  return _heap.serviceability_memory_pool()->get_memory_usage();
+}
+
+GrowableArray<GCMemoryManager*> XCollectedHeap::memory_managers() {
+  GrowableArray<GCMemoryManager*> memory_managers(2);
+  memory_managers.append(_heap.serviceability_cycle_memory_manager());
+  memory_managers.append(_heap.serviceability_pause_memory_manager());
+  return memory_managers;
+}
+
+GrowableArray<MemoryPool*> XCollectedHeap::memory_pools() {
+  GrowableArray<MemoryPool*> memory_pools(1);
+  memory_pools.append(_heap.serviceability_memory_pool());
+  return memory_pools;
+}
+
+void XCollectedHeap::object_iterate(ObjectClosure* cl) {
+  _heap.object_iterate(cl, true /* visit_weaks */);
+}
+
+ParallelObjectIteratorImpl* XCollectedHeap::parallel_object_iterator(uint nworkers) {
+  return _heap.parallel_object_iterator(nworkers, true /* visit_weaks */);
+}
+
+void XCollectedHeap::keep_alive(oop obj) {
+  _heap.keep_alive(obj);
+}
+
+void XCollectedHeap::register_nmethod(nmethod* nm) {
+  XNMethod::register_nmethod(nm);
+}
+
+void XCollectedHeap::unregister_nmethod(nmethod* nm) {
+  XNMethod::unregister_nmethod(nm);
+}
+
+void XCollectedHeap::verify_nmethod(nmethod* nm) {
+  // Does nothing
+}
+
+WorkerThreads* XCollectedHeap::safepoint_workers() {
+  return _runtime_workers.workers();
+}
+
+void XCollectedHeap::gc_threads_do(ThreadClosure* tc) const {
+  tc->do_thread(_director);
+  tc->do_thread(_driver);
+  tc->do_thread(_stat);
+  _heap.threads_do(tc);
+  _runtime_workers.threads_do(tc);
+}
+
+VirtualSpaceSummary XCollectedHeap::create_heap_space_summary() {
+  return VirtualSpaceSummary((HeapWord*)0, (HeapWord*)capacity(), (HeapWord*)max_capacity());
+}
+
+void XCollectedHeap::safepoint_synchronize_begin() {
+  SuspendibleThreadSet::synchronize();
+}
+
+void XCollectedHeap::safepoint_synchronize_end() {
+  SuspendibleThreadSet::desynchronize();
+}
+
+void XCollectedHeap::pin_object(JavaThread* thread, oop obj) {
+  GCLocker::lock_critical(thread);
+}
+
+void XCollectedHeap::unpin_object(JavaThread* thread, oop obj) {
+  GCLocker::unlock_critical(thread);
+}
+
+void XCollectedHeap::prepare_for_verify() {
+  // Does nothing
+}
+
+void XCollectedHeap::print_on(outputStream* st) const {
+  _heap.print_on(st);
+}
+
+void XCollectedHeap::print_on_error(outputStream* st) const {
+  st->print_cr("ZGC Globals:");
+  st->print_cr(" GlobalPhase:       %u (%s)", XGlobalPhase, XGlobalPhaseToString());
+  st->print_cr(" GlobalSeqNum:      %u", XGlobalSeqNum);
+  st->print_cr(" Offset Max:        " SIZE_FORMAT "%s (" PTR_FORMAT ")",
+               byte_size_in_exact_unit(XAddressOffsetMax),
+               exact_unit_for_byte_size(XAddressOffsetMax),
+               XAddressOffsetMax);
+  st->print_cr(" Page Size Small:   " SIZE_FORMAT "M", XPageSizeSmall / M);
+  st->print_cr(" Page Size Medium:  " SIZE_FORMAT "M", XPageSizeMedium / M);
+  st->cr();
+  st->print_cr("ZGC Metadata Bits:");
+  st->print_cr(" Good:              " PTR_FORMAT, XAddressGoodMask);
+  st->print_cr(" Bad:               " PTR_FORMAT, XAddressBadMask);
+  st->print_cr(" WeakBad:           " PTR_FORMAT, XAddressWeakBadMask);
+  st->print_cr(" Marked:            " PTR_FORMAT, XAddressMetadataMarked);
+  st->print_cr(" Remapped:          " PTR_FORMAT, XAddressMetadataRemapped);
+  st->cr();
+  CollectedHeap::print_on_error(st);
+}
+
+void XCollectedHeap::print_extended_on(outputStream* st) const {
+  _heap.print_extended_on(st);
+}
+
+void XCollectedHeap::print_tracing_info() const {
+  // Does nothing
+}
+
+bool XCollectedHeap::print_location(outputStream* st, void* addr) const {
+  return _heap.print_location(st, (uintptr_t)addr);
+}
+
+void XCollectedHeap::verify(VerifyOption option /* ignored */) {
+  _heap.verify();
+}
+
+bool XCollectedHeap::is_oop(oop object) const {
+  return _heap.is_oop(XOop::to_address(object));
+}
+
+bool XCollectedHeap::supports_concurrent_gc_breakpoints() const {
+  return true;
+}
diff --git a/src/hotspot/share/gc/x/xCollectedHeap.hpp b/src/hotspot/share/gc/x/xCollectedHeap.hpp
new file mode 100644
index 00000000000..302963ca2c4
--- /dev/null
+++ b/src/hotspot/share/gc/x/xCollectedHeap.hpp
@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XCOLLECTEDHEAP_HPP
+#define SHARE_GC_X_XCOLLECTEDHEAP_HPP
+
+#include "gc/shared/collectedHeap.hpp"
+#include "gc/shared/softRefPolicy.hpp"
+#include "gc/x/xBarrierSet.hpp"
+#include "gc/x/xHeap.hpp"
+#include "gc/x/xInitialize.hpp"
+#include "gc/x/xRuntimeWorkers.hpp"
+#include "memory/metaspace.hpp"
+#include "services/memoryUsage.hpp"
+
+class VMStructs;
+class XDirector;
+class XDriver;
+class XStat;
+
+class XCollectedHeap : public CollectedHeap {
+  friend class ::VMStructs;
+
+private:
+  SoftRefPolicy     _soft_ref_policy;
+  XBarrierSet       _barrier_set;
+  XInitialize       _initialize;
+  XHeap             _heap;
+  XDriver*          _driver;
+  XDirector*        _director;
+  XStat*            _stat;
+  XRuntimeWorkers   _runtime_workers;
+
+  HeapWord* allocate_new_tlab(size_t min_size,
+                              size_t requested_size,
+                              size_t* actual_size) override;
+
+public:
+  static XCollectedHeap* heap();
+
+  XCollectedHeap();
+  Name kind() const override;
+  const char* name() const override;
+  jint initialize() override;
+  void initialize_serviceability() override;
+  void stop() override;
+
+  SoftRefPolicy* soft_ref_policy() override;
+
+  size_t max_capacity() const override;
+  size_t capacity() const override;
+  size_t used() const override;
+  size_t unused() const override;
+
+  bool is_maximal_no_gc() const override;
+  bool is_in(const void* p) const override;
+  bool requires_barriers(stackChunkOop obj) const override;
+
+  oop array_allocate(Klass* klass, size_t size, int length, bool do_zero, TRAPS) override;
+  HeapWord* mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) override;
+  MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
+                                               size_t size,
+                                               Metaspace::MetadataType mdtype) override;
+  void collect(GCCause::Cause cause) override;
+  void collect_as_vm_thread(GCCause::Cause cause) override;
+  void do_full_collection(bool clear_all_soft_refs) override;
+
+  size_t tlab_capacity(Thread* thr) const override;
+  size_t tlab_used(Thread* thr) const override;
+  size_t max_tlab_size() const override;
+  size_t unsafe_max_tlab_alloc(Thread* thr) const override;
+
+  bool uses_stack_watermark_barrier() const override;
+
+  MemoryUsage memory_usage() override;
+  GrowableArray<GCMemoryManager*> memory_managers() override;
+  GrowableArray<MemoryPool*> memory_pools() override;
+
+  void object_iterate(ObjectClosure* cl) override;
+  ParallelObjectIteratorImpl* parallel_object_iterator(uint nworkers) override;
+
+  void keep_alive(oop obj) override;
+
+  void register_nmethod(nmethod* nm) override;
+  void unregister_nmethod(nmethod* nm) override;
+  void verify_nmethod(nmethod* nmethod) override;
+
+  WorkerThreads* safepoint_workers() override;
+
+  void gc_threads_do(ThreadClosure* tc) const override;
+
+  VirtualSpaceSummary create_heap_space_summary() override;
+
+  void safepoint_synchronize_begin() override;
+  void safepoint_synchronize_end() override;
+
+  void pin_object(JavaThread* thread, oop obj) override;
+  void unpin_object(JavaThread* thread, oop obj) override;
+
+  void print_on(outputStream* st) const override;
+  void print_on_error(outputStream* st) const override;
+  void print_extended_on(outputStream* st) const override;
+  void print_tracing_info() const override;
+  bool print_location(outputStream* st, void* addr) const override;
+
+  void prepare_for_verify() override;
+  void verify(VerifyOption option /* ignored */) override;
+  bool is_oop(oop object) const override;
+  bool supports_concurrent_gc_breakpoints() const override;
+};
+
+#endif // SHARE_GC_X_XCOLLECTEDHEAP_HPP
diff --git a/src/hotspot/share/gc/x/xDebug.gdb b/src/hotspot/share/gc/x/xDebug.gdb
new file mode 100644
index 00000000000..d8f79d589c8
--- /dev/null
+++ b/src/hotspot/share/gc/x/xDebug.gdb
@@ -0,0 +1,148 @@
+#
+# GDB functions for debugging the Z Garbage Collector
+#
+
+printf "Loading zDebug.gdb\n"
+
+# Print Klass*
+define zpk
+    printf "Klass: %s\n", (char*)((Klass*)($arg0))->_name->_body
+end
+
+# Print oop
+define zpo
+    set $obj = (oopDesc*)($arg0)
+
+    printf "Oop:   0x%016llx\tState: ", (uintptr_t)$obj
+    if ((uintptr_t)$obj & (uintptr_t)XAddressGoodMask)
+        printf "Good "
+        if ((uintptr_t)$obj & (uintptr_t)XAddressMetadataRemapped)
+            printf "(Remapped)"
+        else
+            if ((uintptr_t)$obj & (uintptr_t)XAddressMetadataMarked)
+                printf "(Marked)"
+            else
+                printf "(Unknown)"
+            end
+        end
+    else
+        printf "Bad "
+        if ((uintptr_t)XAddressGoodMask & (uintptr_t)XAddressMetadataMarked)
+            # Should be marked
+            if ((uintptr_t)$obj & (uintptr_t)XAddressMetadataRemapped)
+                printf "(Not Marked, Remapped)"
+            else
+                printf "(Not Marked, Not Remapped)"
+            end
+        else
+            if ((uintptr_t)XAddressGoodMask & (uintptr_t)XAddressMetadataRemapped)
+                # Should be remapped
+                if ((uintptr_t)$obj & (uintptr_t)XAddressMetadataMarked)
+                    printf "(Marked, Not Remapped)"
+                else
+                    printf "(Not Marked, Not Remapped)"
+                end
+            else
+                # Unknown
+                printf "(Unknown)"
+            end
+        end
+    end
+    printf "\t Page: %llu\n", ((uintptr_t)$obj & XAddressOffsetMask) >> XGranuleSizeShift
+    x/16gx $obj
+    if (UseCompressedClassPointers)
+        set $klass = (Klass*)(void*)((uintptr_t)CompressedKlassPointers::_narrow_klass._base +((uintptr_t)$obj->_metadata->_compressed_klass << CompressedKlassPointers::_narrow_klass._shift))
+    else
+        set $klass = $obj->_metadata->_klass
+    end
+    printf "Mark:  0x%016llx\tKlass: %s\n", (uintptr_t)$obj->_mark, (char*)$klass->_name->_body
+end
+
+# Print heap page by page table index
+define zpp
+    set $page = (XPage*)((uintptr_t)XHeap::_heap._page_table._map._map[($arg0)] & ~1)
+    printf "Page %p\n", $page
+    print *$page
+end
+
+# Print page_table
+define zpt
+    printf "Pagetable (first 128 slots)\n"
+    x/128gx XHeap::_heap._page_table._map._map
+end
+
+# Print live map
+define __zmarked
+    set $livemap   = $arg0
+    set $bit        = $arg1
+    set $size       = $livemap._bitmap._size
+    set $segment    = $size / XLiveMap::nsegments
+    set $segment_bit = 1 << $segment
+
+    printf "Segment is "
+    if !($livemap._segment_live_bits & $segment_bit)
+        printf "NOT "
+    end
+    printf "live (segment %d)\n", $segment
+
+    if $bit >= $size
+        print "Error: Bit %z out of bounds (bitmap size %z)\n", $bit, $size
+    else
+        set $word_index = $bit / 64
+        set $bit_index  = $bit % 64
+        set $word       = $livemap._bitmap._map[$word_index]
+        set $live_bit   = $word & (1 << $bit_index)
+
+        printf "Object is "
+        if $live_bit == 0
+            printf "NOT "
+        end
+        printf "live (word index %d, bit index %d)\n", $word_index, $bit_index
+    end
+end
+
+define zmarked
+    set $addr          = $arg0
+    set $obj           = ((uintptr_t)$addr & XAddressOffsetMask)
+    set $page_index    = $obj >> XGranuleSizeShift
+    set $page_entry    = (uintptr_t)XHeap::_heap._page_table._map._map[$page_index]
+    set $page          = (XPage*)($page_entry & ~1)
+    set $page_start    = (uintptr_t)$page._virtual._start
+    set $page_end      = (uintptr_t)$page._virtual._end
+    set $page_seqnum   = $page._livemap._seqnum
+    set $global_seqnum = XGlobalSeqNum
+
+    if $obj < $page_start || $obj >= $page_end
+        printf "Error: %p not in page %p (start %p, end %p)\n", $obj, $page, $page_start, $page_end
+    else
+        printf "Page is "
+        if $page_seqnum != $global_seqnum
+            printf "NOT "
+        end
+        printf "live (page %p, page seqnum %d, global seqnum %d)\n", $page, $page_seqnum, $global_seqnum
+
+        #if $page_seqnum == $global_seqnum
+            set $offset = $obj - $page_start
+            set $bit = $offset / 8
+            __zmarked $page._livemap $bit
+        #end
+    end
+end
+
+# Print heap information
+define zph
+    printf "Heap\n"
+    printf "     GlobalPhase:       %u\n", XGlobalPhase
+    printf "     GlobalSeqNum:      %u\n", XGlobalSeqNum
+    printf "     Offset Max:        %-15llu (0x%llx)\n", XAddressOffsetMax, XAddressOffsetMax
+    printf "     Page Size Small:   %-15llu (0x%llx)\n", XPageSizeSmall, XPageSizeSmall
+    printf "     Page Size Medium:  %-15llu (0x%llx)\n", XPageSizeMedium, XPageSizeMedium
+    printf "Metadata Bits\n"
+    printf "     Good:              0x%016llx\n", XAddressGoodMask
+    printf "     Bad:               0x%016llx\n", XAddressBadMask
+    printf "     WeakBad:           0x%016llx\n", XAddressWeakBadMask
+    printf "     Marked:            0x%016llx\n", XAddressMetadataMarked
+    printf "     Remapped:          0x%016llx\n", XAddressMetadataRemapped
+end
+
+# End of file
diff --git a/src/hotspot/share/gc/x/xDirector.cpp b/src/hotspot/share/gc/x/xDirector.cpp
new file mode 100644
index 00000000000..05ef3623741
--- /dev/null
+++ b/src/hotspot/share/gc/x/xDirector.cpp
@@ -0,0 +1,406 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/xDirector.hpp"
+#include "gc/x/xDriver.hpp"
+#include "gc/x/xHeap.inline.hpp"
+#include "gc/x/xHeuristics.hpp"
+#include "gc/x/xStat.hpp"
+#include "logging/log.hpp"
+
+constexpr double one_in_1000 = 3.290527;
+constexpr double sample_interval = 1.0 / XStatAllocRate::sample_hz;
+
+XDirector::XDirector(XDriver* driver) :
+    _driver(driver),
+    _metronome(XStatAllocRate::sample_hz) {
+  set_name("XDirector");
+  create_and_start();
+}
+
+static void sample_allocation_rate() {
+  // Sample allocation rate. This is needed by rule_allocation_rate()
+  // below to estimate the time we have until we run out of memory.
+  const double bytes_per_second = XStatAllocRate::sample_and_reset();
+
+  log_debug(gc, alloc)("Allocation Rate: %.1fMB/s, Predicted: %.1fMB/s, Avg: %.1f(+/-%.1f)MB/s",
+                       bytes_per_second / M,
+                       XStatAllocRate::predict() / M,
+                       XStatAllocRate::avg() / M,
+                       XStatAllocRate::sd() / M);
+}
+
+static XDriverRequest rule_allocation_stall() {
+  // Perform GC if we've observed at least one allocation stall since
+  // the last GC started.
+  if (!XHeap::heap()->has_alloc_stalled()) {
+    return GCCause::_no_gc;
+  }
+
+  log_debug(gc, director)("Rule: Allocation Stall Observed");
+
+  return GCCause::_z_allocation_stall;
+}
+
+static XDriverRequest rule_warmup() {
+  if (XStatCycle::is_warm()) {
+    // Rule disabled
+    return GCCause::_no_gc;
+  }
+
+  // Perform GC if heap usage passes 10/20/30% and no other GC has been
+  // performed yet. This allows us to get some early samples of the GC
+  // duration, which is needed by the other rules.
+  const size_t soft_max_capacity = XHeap::heap()->soft_max_capacity();
+  const size_t used = XHeap::heap()->used();
+  const double used_threshold_percent = (XStatCycle::nwarmup_cycles() + 1) * 0.1;
+  const size_t used_threshold = soft_max_capacity * used_threshold_percent;
+
+  log_debug(gc, director)("Rule: Warmup %.0f%%, Used: " SIZE_FORMAT "MB, UsedThreshold: " SIZE_FORMAT "MB",
+                          used_threshold_percent * 100, used / M, used_threshold / M);
+
+  if (used < used_threshold) {
+    return GCCause::_no_gc;
+  }
+
+  return GCCause::_z_warmup;
+}
+
+static XDriverRequest rule_timer() {
+  if (ZCollectionInterval <= 0) {
+    // Rule disabled
+    return GCCause::_no_gc;
+  }
+
+  // Perform GC if timer has expired.
+  const double time_since_last_gc = XStatCycle::time_since_last();
+  const double time_until_gc = ZCollectionInterval - time_since_last_gc;
+
+  log_debug(gc, director)("Rule: Timer, Interval: %.3fs, TimeUntilGC: %.3fs",
+                          ZCollectionInterval, time_until_gc);
+
+  if (time_until_gc > 0) {
+    return GCCause::_no_gc;
+  }
+
+  return GCCause::_z_timer;
+}
+
+static double estimated_gc_workers(double serial_gc_time, double parallelizable_gc_time, double time_until_deadline) {
+  const double parallelizable_time_until_deadline = MAX2(time_until_deadline - serial_gc_time, 0.001);
+  return parallelizable_gc_time / parallelizable_time_until_deadline;
+}
+
+static uint discrete_gc_workers(double gc_workers) {
+  return clamp<uint>(ceil(gc_workers), 1, ConcGCThreads);
+}
+
+static double select_gc_workers(double serial_gc_time, double parallelizable_gc_time, double alloc_rate_sd_percent, double time_until_oom) {
+  // Use all workers until we're warm
+  if (!XStatCycle::is_warm()) {
+    const double not_warm_gc_workers = ConcGCThreads;
+    log_debug(gc, director)("Select GC Workers (Not Warm), GCWorkers: %.3f", not_warm_gc_workers);
+    return not_warm_gc_workers;
+  }
+
+  // Calculate number of GC workers needed to avoid a long GC cycle and to avoid OOM.
+  const double avoid_long_gc_workers = estimated_gc_workers(serial_gc_time, parallelizable_gc_time, 10 /* seconds */);
+  const double avoid_oom_gc_workers = estimated_gc_workers(serial_gc_time, parallelizable_gc_time, time_until_oom);
+
+  const double gc_workers = MAX2(avoid_long_gc_workers, avoid_oom_gc_workers);
+  const uint actual_gc_workers = discrete_gc_workers(gc_workers);
+  const uint last_gc_workers = XStatCycle::last_active_workers();
+
+  // More than 15% division from the average is considered unsteady
+  if (alloc_rate_sd_percent >= 0.15) {
+    const double half_gc_workers = ConcGCThreads / 2.0;
+    const double unsteady_gc_workers = MAX3<double>(gc_workers, last_gc_workers, half_gc_workers);
+    log_debug(gc, director)("Select GC Workers (Unsteady), "
+                            "AvoidLongGCWorkers: %.3f, AvoidOOMGCWorkers: %.3f, LastGCWorkers: %.3f, HalfGCWorkers: %.3f, GCWorkers: %.3f",
+                            avoid_long_gc_workers, avoid_oom_gc_workers, (double)last_gc_workers, half_gc_workers, unsteady_gc_workers);
+    return unsteady_gc_workers;
+  }
+
+  if (actual_gc_workers < last_gc_workers) {
+    // Before decreasing number of GC workers compared to the previous GC cycle, check if the
+    // next GC cycle will need to increase it again. If so, use the same number of GC workers
+    // that will be needed in the next cycle.
+    const double gc_duration_delta = (parallelizable_gc_time / actual_gc_workers) - (parallelizable_gc_time / last_gc_workers);
+    const double additional_time_for_allocations = XStatCycle::time_since_last() - gc_duration_delta - sample_interval;
+    const double next_time_until_oom = time_until_oom + additional_time_for_allocations;
+    const double next_avoid_oom_gc_workers = estimated_gc_workers(serial_gc_time, parallelizable_gc_time, next_time_until_oom);
+
+    // Add 0.5 to increase friction and avoid lowering too eagerly
+    const double next_gc_workers = next_avoid_oom_gc_workers + 0.5;
+    const double try_lowering_gc_workers = clamp<double>(next_gc_workers, actual_gc_workers, last_gc_workers);
+
+    log_debug(gc, director)("Select GC Workers (Try Lowering), "
+                           "AvoidLongGCWorkers: %.3f, AvoidOOMGCWorkers: %.3f, NextAvoidOOMGCWorkers: %.3f, LastGCWorkers: %.3f, GCWorkers: %.3f",
+                            avoid_long_gc_workers, avoid_oom_gc_workers, next_avoid_oom_gc_workers, (double)last_gc_workers, try_lowering_gc_workers);
+    return try_lowering_gc_workers;
+  }
+
+  log_debug(gc, director)("Select GC Workers (Normal), "
+                         "AvoidLongGCWorkers: %.3f, AvoidOOMGCWorkers: %.3f, LastGCWorkers: %.3f, GCWorkers: %.3f",
+                         avoid_long_gc_workers, avoid_oom_gc_workers, (double)last_gc_workers, gc_workers);
+  return gc_workers;
+}
+
+XDriverRequest rule_allocation_rate_dynamic() {
+  if (!XStatCycle::is_time_trustable()) {
+    // Rule disabled
+    return GCCause::_no_gc;
+  }
+
+  // Calculate amount of free memory available. Note that we take the
+  // relocation headroom into account to avoid in-place relocation.
+  const size_t soft_max_capacity = XHeap::heap()->soft_max_capacity();
+  const size_t used = XHeap::heap()->used();
+  const size_t free_including_headroom = soft_max_capacity - MIN2(soft_max_capacity, used);
+  const size_t free = free_including_headroom - MIN2(free_including_headroom, XHeuristics::relocation_headroom());
+
+  // Calculate time until OOM given the max allocation rate and the amount
+  // of free memory. The allocation rate is a moving average and we multiply
+  // that with an allocation spike tolerance factor to guard against unforeseen
+  // phase changes in the allocate rate. We then add ~3.3 sigma to account for
+  // the allocation rate variance, which means the probability is 1 in 1000
+  // that a sample is outside of the confidence interval.
+  const double alloc_rate_predict = XStatAllocRate::predict();
+  const double alloc_rate_avg = XStatAllocRate::avg();
+  const double alloc_rate_sd = XStatAllocRate::sd();
+  const double alloc_rate_sd_percent = alloc_rate_sd / (alloc_rate_avg + 1.0);
+  const double alloc_rate = (MAX2(alloc_rate_predict, alloc_rate_avg) * ZAllocationSpikeTolerance) + (alloc_rate_sd * one_in_1000) + 1.0;
+  const double time_until_oom = (free / alloc_rate) / (1.0 + alloc_rate_sd_percent);
+
+  // Calculate max serial/parallel times of a GC cycle. The times are
+  // moving averages, we add ~3.3 sigma to account for the variance.
+  const double serial_gc_time = XStatCycle::serial_time().davg() + (XStatCycle::serial_time().dsd() * one_in_1000);
+  const double parallelizable_gc_time = XStatCycle::parallelizable_time().davg() + (XStatCycle::parallelizable_time().dsd() * one_in_1000);
+
+  // Calculate number of GC workers needed to avoid OOM.
+  const double gc_workers = select_gc_workers(serial_gc_time, parallelizable_gc_time, alloc_rate_sd_percent, time_until_oom);
+
+  // Convert to a discrete number of GC workers within limits.
+  const uint actual_gc_workers = discrete_gc_workers(gc_workers);
+
+  // Calculate GC duration given number of GC workers needed.
+  const double actual_gc_duration = serial_gc_time + (parallelizable_gc_time / actual_gc_workers);
+  const uint last_gc_workers = XStatCycle::last_active_workers();
+
+  // Calculate time until GC given the time until OOM and GC duration.
+  // We also subtract the sample interval, so that we don't overshoot the
+  // target time and end up starting the GC too late in the next interval.
+  const double time_until_gc = time_until_oom - actual_gc_duration - sample_interval;
+
+  log_debug(gc, director)("Rule: Allocation Rate (Dynamic GC Workers), "
+                          "MaxAllocRate: %.1fMB/s (+/-%.1f%%), Free: " SIZE_FORMAT "MB, GCCPUTime: %.3f, "
+                          "GCDuration: %.3fs, TimeUntilOOM: %.3fs, TimeUntilGC: %.3fs, GCWorkers: %u -> %u",
+                          alloc_rate / M,
+                          alloc_rate_sd_percent * 100,
+                          free / M,
+                          serial_gc_time + parallelizable_gc_time,
+                          serial_gc_time + (parallelizable_gc_time / actual_gc_workers),
+                          time_until_oom,
+                          time_until_gc,
+                          last_gc_workers,
+                          actual_gc_workers);
+
+  if (actual_gc_workers <= last_gc_workers && time_until_gc > 0) {
+    return XDriverRequest(GCCause::_no_gc, actual_gc_workers);
+  }
+
+  return XDriverRequest(GCCause::_z_allocation_rate, actual_gc_workers);
+}
+
+static XDriverRequest rule_allocation_rate_static() {
+  if (!XStatCycle::is_time_trustable()) {
+    // Rule disabled
+    return GCCause::_no_gc;
+  }
+
+  // Perform GC if the estimated max allocation rate indicates that we
+  // will run out of memory. The estimated max allocation rate is based
+  // on the moving average of the sampled allocation rate plus a safety
+  // margin based on variations in the allocation rate and unforeseen
+  // allocation spikes.
+
+  // Calculate amount of free memory available. Note that we take the
+  // relocation headroom into account to avoid in-place relocation.
+  const size_t soft_max_capacity = XHeap::heap()->soft_max_capacity();
+  const size_t used = XHeap::heap()->used();
+  const size_t free_including_headroom = soft_max_capacity - MIN2(soft_max_capacity, used);
+  const size_t free = free_including_headroom - MIN2(free_including_headroom, XHeuristics::relocation_headroom());
+
+  // Calculate time until OOM given the max allocation rate and the amount
+  // of free memory. The allocation rate is a moving average and we multiply
+  // that with an allocation spike tolerance factor to guard against unforeseen
+  // phase changes in the allocate rate. We then add ~3.3 sigma to account for
+  // the allocation rate variance, which means the probability is 1 in 1000
+  // that a sample is outside of the confidence interval.
+  const double max_alloc_rate = (XStatAllocRate::avg() * ZAllocationSpikeTolerance) + (XStatAllocRate::sd() * one_in_1000);
+  const double time_until_oom = free / (max_alloc_rate + 1.0); // Plus 1.0B/s to avoid division by zero
+
+  // Calculate max serial/parallel times of a GC cycle. The times are
+  // moving averages, we add ~3.3 sigma to account for the variance.
+  const double serial_gc_time = XStatCycle::serial_time().davg() + (XStatCycle::serial_time().dsd() * one_in_1000);
+  const double parallelizable_gc_time = XStatCycle::parallelizable_time().davg() + (XStatCycle::parallelizable_time().dsd() * one_in_1000);
+
+  // Calculate GC duration given number of GC workers needed.
+  const double gc_duration = serial_gc_time + (parallelizable_gc_time / ConcGCThreads);
+
+  // Calculate time until GC given the time until OOM and max duration of GC.
+  // We also deduct the sample interval, so that we don't overshoot the target
+  // time and end up starting the GC too late in the next interval.
+  const double time_until_gc = time_until_oom - gc_duration - sample_interval;
+
+  log_debug(gc, director)("Rule: Allocation Rate (Static GC Workers), MaxAllocRate: %.1fMB/s, Free: " SIZE_FORMAT "MB, GCDuration: %.3fs, TimeUntilGC: %.3fs",
+                          max_alloc_rate / M, free / M, gc_duration, time_until_gc);
+
+  if (time_until_gc > 0) {
+    return GCCause::_no_gc;
+  }
+
+  return GCCause::_z_allocation_rate;
+}
+
+static XDriverRequest rule_allocation_rate() {
+  if (UseDynamicNumberOfGCThreads) {
+    return rule_allocation_rate_dynamic();
+  } else {
+    return rule_allocation_rate_static();
+  }
+}
+
+static XDriverRequest rule_high_usage() {
+  // Perform GC if the amount of free memory is 5% or less. This is a preventive
+  // meassure in the case where the application has a very low allocation rate,
+  // such that the allocation rate rule doesn't trigger, but the amount of free
+  // memory is still slowly but surely heading towards zero. In this situation,
+  // we start a GC cycle to avoid a potential allocation stall later.
+
+  // Calculate amount of free memory available. Note that we take the
+  // relocation headroom into account to avoid in-place relocation.
+  const size_t soft_max_capacity = XHeap::heap()->soft_max_capacity();
+  const size_t used = XHeap::heap()->used();
+  const size_t free_including_headroom = soft_max_capacity - MIN2(soft_max_capacity, used);
+  const size_t free = free_including_headroom - MIN2(free_including_headroom, XHeuristics::relocation_headroom());
+  const double free_percent = percent_of(free, soft_max_capacity);
+
+  log_debug(gc, director)("Rule: High Usage, Free: " SIZE_FORMAT "MB(%.1f%%)",
+                          free / M, free_percent);
+
+  if (free_percent > 5.0) {
+    return GCCause::_no_gc;
+  }
+
+  return GCCause::_z_high_usage;
+}
+
+static XDriverRequest rule_proactive() {
+  if (!ZProactive || !XStatCycle::is_warm()) {
+    // Rule disabled
+    return GCCause::_no_gc;
+  }
+
+  // Perform GC if the impact of doing so, in terms of application throughput
+  // reduction, is considered acceptable. This rule allows us to keep the heap
+  // size down and allow reference processing to happen even when we have a lot
+  // of free space on the heap.
+
+  // Only consider doing a proactive GC if the heap usage has grown by at least
+  // 10% of the max capacity since the previous GC, or more than 5 minutes has
+  // passed since the previous GC. This helps avoid superfluous GCs when running
+  // applications with very low allocation rate.
+  const size_t used_after_last_gc = XStatHeap::used_at_relocate_end();
+  const size_t used_increase_threshold = XHeap::heap()->soft_max_capacity() * 0.10; // 10%
+  const size_t used_threshold = used_after_last_gc + used_increase_threshold;
+  const size_t used = XHeap::heap()->used();
+  const double time_since_last_gc = XStatCycle::time_since_last();
+  const double time_since_last_gc_threshold = 5 * 60; // 5 minutes
+  if (used < used_threshold && time_since_last_gc < time_since_last_gc_threshold) {
+    // Don't even consider doing a proactive GC
+    log_debug(gc, director)("Rule: Proactive, UsedUntilEnabled: " SIZE_FORMAT "MB, TimeUntilEnabled: %.3fs",
+                            (used_threshold - used) / M,
+                            time_since_last_gc_threshold - time_since_last_gc);
+    return GCCause::_no_gc;
+  }
+
+  const double assumed_throughput_drop_during_gc = 0.50; // 50%
+  const double acceptable_throughput_drop = 0.01;        // 1%
+  const double serial_gc_time = XStatCycle::serial_time().davg() + (XStatCycle::serial_time().dsd() * one_in_1000);
+  const double parallelizable_gc_time = XStatCycle::parallelizable_time().davg() + (XStatCycle::parallelizable_time().dsd() * one_in_1000);
+  const double gc_duration = serial_gc_time + (parallelizable_gc_time / ConcGCThreads);
+  const double acceptable_gc_interval = gc_duration * ((assumed_throughput_drop_during_gc / acceptable_throughput_drop) - 1.0);
+  const double time_until_gc = acceptable_gc_interval - time_since_last_gc;
+
+  log_debug(gc, director)("Rule: Proactive, AcceptableGCInterval: %.3fs, TimeSinceLastGC: %.3fs, TimeUntilGC: %.3fs",
+                          acceptable_gc_interval, time_since_last_gc, time_until_gc);
+
+  if (time_until_gc > 0) {
+    return GCCause::_no_gc;
+  }
+
+  return GCCause::_z_proactive;
+}
+
+static XDriverRequest make_gc_decision() {
+  // List of rules
+  using XDirectorRule = XDriverRequest (*)();
+  const XDirectorRule rules[] = {
+    rule_allocation_stall,
+    rule_warmup,
+    rule_timer,
+    rule_allocation_rate,
+    rule_high_usage,
+    rule_proactive,
+  };
+
+  // Execute rules
+  for (size_t i = 0; i < ARRAY_SIZE(rules); i++) {
+    const XDriverRequest request = rules[i]();
+    if (request.cause() != GCCause::_no_gc) {
+      return request;
+    }
+  }
+
+  return GCCause::_no_gc;
+}
+
+void XDirector::run_service() {
+  // Main loop
+  while (_metronome.wait_for_tick()) {
+    sample_allocation_rate();
+    if (!_driver->is_busy()) {
+      const XDriverRequest request = make_gc_decision();
+      if (request.cause() != GCCause::_no_gc) {
+        _driver->collect(request);
+      }
+    }
+  }
+}
+
+void XDirector::stop_service() {
+  _metronome.stop();
+}
diff --git a/src/hotspot/share/gc/x/xDirector.hpp b/src/hotspot/share/gc/x/xDirector.hpp
new file mode 100644
index 00000000000..eacce20e8c9
--- /dev/null
+++ b/src/hotspot/share/gc/x/xDirector.hpp
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XDIRECTOR_HPP
+#define SHARE_GC_X_XDIRECTOR_HPP
+
+#include "gc/shared/concurrentGCThread.hpp"
+#include "gc/x/xMetronome.hpp"
+
+class XDriver;
+
+class XDirector : public ConcurrentGCThread {
+private:
+  XDriver* const _driver;
+  XMetronome     _metronome;
+
+protected:
+  virtual void run_service();
+  virtual void stop_service();
+
+public:
+  XDirector(XDriver* driver);
+};
+
+#endif // SHARE_GC_X_XDIRECTOR_HPP
diff --git a/src/hotspot/share/gc/x/xDriver.cpp b/src/hotspot/share/gc/x/xDriver.cpp
new file mode 100644
index 00000000000..f76d9f4e586
--- /dev/null
+++ b/src/hotspot/share/gc/x/xDriver.cpp
@@ -0,0 +1,513 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcId.hpp"
+#include "gc/shared/gcLocker.hpp"
+#include "gc/shared/gcVMOperations.hpp"
+#include "gc/shared/isGCActiveMark.hpp"
+#include "gc/x/xAbort.inline.hpp"
+#include "gc/x/xBreakpoint.hpp"
+#include "gc/x/xCollectedHeap.hpp"
+#include "gc/x/xDriver.hpp"
+#include "gc/x/xHeap.inline.hpp"
+#include "gc/x/xMessagePort.inline.hpp"
+#include "gc/x/xServiceability.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xVerify.hpp"
+#include "logging/log.hpp"
+#include "memory/universe.hpp"
+#include "runtime/threads.hpp"
+#include "runtime/vmOperations.hpp"
+#include "runtime/vmThread.hpp"
+
+static const XStatPhaseCycle      XPhaseCycle("Garbage Collection Cycle");
+static const XStatPhasePause      XPhasePauseMarkStart("Pause Mark Start");
+static const XStatPhaseConcurrent XPhaseConcurrentMark("Concurrent Mark");
+static const XStatPhaseConcurrent XPhaseConcurrentMarkContinue("Concurrent Mark Continue");
+static const XStatPhaseConcurrent XPhaseConcurrentMarkFree("Concurrent Mark Free");
+static const XStatPhasePause      XPhasePauseMarkEnd("Pause Mark End");
+static const XStatPhaseConcurrent XPhaseConcurrentProcessNonStrongReferences("Concurrent Process Non-Strong References");
+static const XStatPhaseConcurrent XPhaseConcurrentResetRelocationSet("Concurrent Reset Relocation Set");
+static const XStatPhaseConcurrent XPhaseConcurrentSelectRelocationSet("Concurrent Select Relocation Set");
+static const XStatPhasePause      XPhasePauseRelocateStart("Pause Relocate Start");
+static const XStatPhaseConcurrent XPhaseConcurrentRelocated("Concurrent Relocate");
+static const XStatCriticalPhase   XCriticalPhaseGCLockerStall("GC Locker Stall", false /* verbose */);
+static const XStatSampler         XSamplerJavaThreads("System", "Java Threads", XStatUnitThreads);
+
+XDriverRequest::XDriverRequest() :
+    XDriverRequest(GCCause::_no_gc) {}
+
+XDriverRequest::XDriverRequest(GCCause::Cause cause) :
+    XDriverRequest(cause, ConcGCThreads) {}
+
+XDriverRequest::XDriverRequest(GCCause::Cause cause, uint nworkers) :
+    _cause(cause),
+    _nworkers(nworkers) {}
+
+bool XDriverRequest::operator==(const XDriverRequest& other) const {
+  return _cause == other._cause;
+}
+
+GCCause::Cause XDriverRequest::cause() const {
+  return _cause;
+}
+
+uint XDriverRequest::nworkers() const {
+  return _nworkers;
+}
+
+class VM_XOperation : public VM_Operation {
+private:
+  const uint _gc_id;
+  bool       _gc_locked;
+  bool       _success;
+
+public:
+  VM_XOperation() :
+      _gc_id(GCId::current()),
+      _gc_locked(false),
+      _success(false) {}
+
+  virtual bool needs_inactive_gc_locker() const {
+    // An inactive GC locker is needed in operations where we change the bad
+    // mask or move objects. Changing the bad mask will invalidate all oops,
+    // which makes it conceptually the same thing as moving all objects.
+    return false;
+  }
+
+  virtual bool skip_thread_oop_barriers() const {
+    return true;
+  }
+
+  virtual bool do_operation() = 0;
+
+  virtual bool doit_prologue() {
+    Heap_lock->lock();
+    return true;
+  }
+
+  virtual void doit() {
+    // Abort if GC locker state is incompatible
+    if (needs_inactive_gc_locker() && GCLocker::check_active_before_gc()) {
+      _gc_locked = true;
+      return;
+    }
+
+    // Setup GC id and active marker
+    GCIdMark gc_id_mark(_gc_id);
+    IsGCActiveMark gc_active_mark;
+
+    // Verify before operation
+    XVerify::before_zoperation();
+
+    // Execute operation
+    _success = do_operation();
+
+    // Update statistics
+    XStatSample(XSamplerJavaThreads, Threads::number_of_threads());
+  }
+
+  virtual void doit_epilogue() {
+    Heap_lock->unlock();
+  }
+
+  bool gc_locked() const {
+    return _gc_locked;
+  }
+
+  bool success() const {
+    return _success;
+  }
+};
+
+class VM_XMarkStart : public VM_XOperation {
+public:
+  virtual VMOp_Type type() const {
+    return VMOp_XMarkStart;
+  }
+
+  virtual bool needs_inactive_gc_locker() const {
+    return true;
+  }
+
+  virtual bool do_operation() {
+    XStatTimer timer(XPhasePauseMarkStart);
+    XServiceabilityPauseTracer tracer;
+
+    XCollectedHeap::heap()->increment_total_collections(true /* full */);
+
+    XHeap::heap()->mark_start();
+    return true;
+  }
+};
+
+class VM_XMarkEnd : public VM_XOperation {
+public:
+  virtual VMOp_Type type() const {
+    return VMOp_XMarkEnd;
+  }
+
+  virtual bool do_operation() {
+    XStatTimer timer(XPhasePauseMarkEnd);
+    XServiceabilityPauseTracer tracer;
+    return XHeap::heap()->mark_end();
+  }
+};
+
+class VM_XRelocateStart : public VM_XOperation {
+public:
+  virtual VMOp_Type type() const {
+    return VMOp_XRelocateStart;
+  }
+
+  virtual bool needs_inactive_gc_locker() const {
+    return true;
+  }
+
+  virtual bool do_operation() {
+    XStatTimer timer(XPhasePauseRelocateStart);
+    XServiceabilityPauseTracer tracer;
+    XHeap::heap()->relocate_start();
+    return true;
+  }
+};
+
+class VM_XVerify : public VM_Operation {
+public:
+  virtual VMOp_Type type() const {
+    return VMOp_XVerify;
+  }
+
+  virtual bool skip_thread_oop_barriers() const {
+    return true;
+  }
+
+  virtual void doit() {
+    XVerify::after_weak_processing();
+  }
+};
+
+XDriver::XDriver() :
+    _gc_cycle_port(),
+    _gc_locker_port() {
+  set_name("XDriver");
+  create_and_start();
+}
+
+bool XDriver::is_busy() const {
+  return _gc_cycle_port.is_busy();
+}
+
+void XDriver::collect(const XDriverRequest& request) {
+  switch (request.cause()) {
+  case GCCause::_heap_dump:
+  case GCCause::_heap_inspection:
+  case GCCause::_wb_young_gc:
+  case GCCause::_wb_full_gc:
+  case GCCause::_dcmd_gc_run:
+  case GCCause::_java_lang_system_gc:
+  case GCCause::_full_gc_alot:
+  case GCCause::_scavenge_alot:
+  case GCCause::_jvmti_force_gc:
+  case GCCause::_metadata_GC_clear_soft_refs:
+  case GCCause::_codecache_GC_aggressive:
+    // Start synchronous GC
+    _gc_cycle_port.send_sync(request);
+    break;
+
+  case GCCause::_z_timer:
+  case GCCause::_z_warmup:
+  case GCCause::_z_allocation_rate:
+  case GCCause::_z_allocation_stall:
+  case GCCause::_z_proactive:
+  case GCCause::_z_high_usage:
+  case GCCause::_codecache_GC_threshold:
+  case GCCause::_metadata_GC_threshold:
+    // Start asynchronous GC
+    _gc_cycle_port.send_async(request);
+    break;
+
+  case GCCause::_gc_locker:
+    // Restart VM operation previously blocked by the GC locker
+    _gc_locker_port.signal();
+    break;
+
+  case GCCause::_wb_breakpoint:
+    XBreakpoint::start_gc();
+    _gc_cycle_port.send_async(request);
+    break;
+
+  default:
+    // Other causes not supported
+    fatal("Unsupported GC cause (%s)", GCCause::to_string(request.cause()));
+    break;
+  }
+}
+
+template <typename T>
+bool XDriver::pause() {
+  for (;;) {
+    T op;
+    VMThread::execute(&op);
+    if (op.gc_locked()) {
+      // Wait for GC to become unlocked and restart the VM operation
+      XStatTimer timer(XCriticalPhaseGCLockerStall);
+      _gc_locker_port.wait();
+      continue;
+    }
+
+    // Notify VM operation completed
+    _gc_locker_port.ack();
+
+    return op.success();
+  }
+}
+
+void XDriver::pause_mark_start() {
+  pause<VM_XMarkStart>();
+}
+
+void XDriver::concurrent_mark() {
+  XStatTimer timer(XPhaseConcurrentMark);
+  XBreakpoint::at_after_marking_started();
+  XHeap::heap()->mark(true /* initial */);
+  XBreakpoint::at_before_marking_completed();
+}
+
+bool XDriver::pause_mark_end() {
+  return pause<VM_XMarkEnd>();
+}
+
+void XDriver::concurrent_mark_continue() {
+  XStatTimer timer(XPhaseConcurrentMarkContinue);
+  XHeap::heap()->mark(false /* initial */);
+}
+
+void XDriver::concurrent_mark_free() {
+  XStatTimer timer(XPhaseConcurrentMarkFree);
+  XHeap::heap()->mark_free();
+}
+
+void XDriver::concurrent_process_non_strong_references() {
+  XStatTimer timer(XPhaseConcurrentProcessNonStrongReferences);
+  XBreakpoint::at_after_reference_processing_started();
+  XHeap::heap()->process_non_strong_references();
+}
+
+void XDriver::concurrent_reset_relocation_set() {
+  XStatTimer timer(XPhaseConcurrentResetRelocationSet);
+  XHeap::heap()->reset_relocation_set();
+}
+
+void XDriver::pause_verify() {
+  if (ZVerifyRoots || ZVerifyObjects) {
+    VM_XVerify op;
+    VMThread::execute(&op);
+  }
+}
+
+void XDriver::concurrent_select_relocation_set() {
+  XStatTimer timer(XPhaseConcurrentSelectRelocationSet);
+  XHeap::heap()->select_relocation_set();
+}
+
+void XDriver::pause_relocate_start() {
+  pause<VM_XRelocateStart>();
+}
+
+void XDriver::concurrent_relocate() {
+  XStatTimer timer(XPhaseConcurrentRelocated);
+  XHeap::heap()->relocate();
+}
+
+void XDriver::check_out_of_memory() {
+  XHeap::heap()->check_out_of_memory();
+}
+
+static bool should_clear_soft_references(const XDriverRequest& request) {
+  // Clear soft references if implied by the GC cause
+  if (request.cause() == GCCause::_wb_full_gc ||
+      request.cause() == GCCause::_metadata_GC_clear_soft_refs ||
+      request.cause() == GCCause::_z_allocation_stall) {
+    // Clear
+    return true;
+  }
+
+  // Don't clear
+  return false;
+}
+
+static uint select_active_worker_threads_dynamic(const XDriverRequest& request) {
+  // Use requested number of worker threads
+  return request.nworkers();
+}
+
+static uint select_active_worker_threads_static(const XDriverRequest& request) {
+  const GCCause::Cause cause = request.cause();
+  const uint nworkers = request.nworkers();
+
+  // Boost number of worker threads if implied by the GC cause
+  if (cause == GCCause::_wb_full_gc ||
+      cause == GCCause::_java_lang_system_gc ||
+      cause == GCCause::_metadata_GC_clear_soft_refs ||
+      cause == GCCause::_z_allocation_stall) {
+    // Boost
+    const uint boosted_nworkers = MAX2(nworkers, ParallelGCThreads);
+    return boosted_nworkers;
+  }
+
+  // Use requested number of worker threads
+  return nworkers;
+}
+
+static uint select_active_worker_threads(const XDriverRequest& request) {
+  if (UseDynamicNumberOfGCThreads) {
+    return select_active_worker_threads_dynamic(request);
+  } else {
+    return select_active_worker_threads_static(request);
+  }
+}
+
+class XDriverGCScope : public StackObj {
+private:
+  GCIdMark                   _gc_id;
+  GCCause::Cause             _gc_cause;
+  GCCauseSetter              _gc_cause_setter;
+  XStatTimer                 _timer;
+  XServiceabilityCycleTracer _tracer;
+
+public:
+  XDriverGCScope(const XDriverRequest& request) :
+      _gc_id(),
+      _gc_cause(request.cause()),
+      _gc_cause_setter(XCollectedHeap::heap(), _gc_cause),
+      _timer(XPhaseCycle),
+      _tracer() {
+    // Update statistics
+    XStatCycle::at_start();
+
+    // Set up soft reference policy
+    const bool clear = should_clear_soft_references(request);
+    XHeap::heap()->set_soft_reference_policy(clear);
+
+    // Select number of worker threads to use
+    const uint nworkers = select_active_worker_threads(request);
+    XHeap::heap()->set_active_workers(nworkers);
+  }
+
+  ~XDriverGCScope() {
+    // Update statistics
+    XStatCycle::at_end(_gc_cause, XHeap::heap()->active_workers());
+
+    // Update data used by soft reference policy
+    Universe::heap()->update_capacity_and_used_at_gc();
+
+    // Signal that we have completed a visit to all live objects
+    Universe::heap()->record_whole_heap_examined_timestamp();
+  }
+};
+
+// Macro to execute a termination check after a concurrent phase. Note
+// that it's important that the termination check comes after the call
+// to the function f, since we can't abort between pause_relocate_start()
+// and concurrent_relocate(). We need to let concurrent_relocate() call
+// abort_page() on the remaining entries in the relocation set.
+#define concurrent(f)                 \
+  do {                                \
+    concurrent_##f();                 \
+    if (should_terminate()) {         \
+      return;                         \
+    }                                 \
+  } while (false)
+
+void XDriver::gc(const XDriverRequest& request) {
+  XDriverGCScope scope(request);
+
+  // Phase 1: Pause Mark Start
+  pause_mark_start();
+
+  // Phase 2: Concurrent Mark
+  concurrent(mark);
+
+  // Phase 3: Pause Mark End
+  while (!pause_mark_end()) {
+    // Phase 3.5: Concurrent Mark Continue
+    concurrent(mark_continue);
+  }
+
+  // Phase 4: Concurrent Mark Free
+  concurrent(mark_free);
+
+  // Phase 5: Concurrent Process Non-Strong References
+  concurrent(process_non_strong_references);
+
+  // Phase 6: Concurrent Reset Relocation Set
+  concurrent(reset_relocation_set);
+
+  // Phase 7: Pause Verify
+  pause_verify();
+
+  // Phase 8: Concurrent Select Relocation Set
+  concurrent(select_relocation_set);
+
+  // Phase 9: Pause Relocate Start
+  pause_relocate_start();
+
+  // Phase 10: Concurrent Relocate
+  concurrent(relocate);
+}
+
+void XDriver::run_service() {
+  // Main loop
+  while (!should_terminate()) {
+    // Wait for GC request
+    const XDriverRequest request = _gc_cycle_port.receive();
+    if (request.cause() == GCCause::_no_gc) {
+      continue;
+    }
+
+    XBreakpoint::at_before_gc();
+
+    // Run GC
+    gc(request);
+
+    if (should_terminate()) {
+      // Abort
+      break;
+    }
+
+    // Notify GC completed
+    _gc_cycle_port.ack();
+
+    // Check for out of memory condition
+    check_out_of_memory();
+
+    XBreakpoint::at_after_gc();
+  }
+}
+
+void XDriver::stop_service() {
+  XAbort::abort();
+  _gc_cycle_port.send_async(GCCause::_no_gc);
+}
diff --git a/src/hotspot/share/gc/x/xDriver.hpp b/src/hotspot/share/gc/x/xDriver.hpp
new file mode 100644
index 00000000000..3803b699b85
--- /dev/null
+++ b/src/hotspot/share/gc/x/xDriver.hpp
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XDRIVER_HPP
+#define SHARE_GC_X_XDRIVER_HPP
+
+#include "gc/shared/concurrentGCThread.hpp"
+#include "gc/shared/gcCause.hpp"
+#include "gc/x/xMessagePort.hpp"
+
+class VM_XOperation;
+
+class XDriverRequest {
+private:
+  GCCause::Cause _cause;
+  uint           _nworkers;
+
+public:
+  XDriverRequest();
+  XDriverRequest(GCCause::Cause cause);
+  XDriverRequest(GCCause::Cause cause, uint nworkers);
+
+  bool operator==(const XDriverRequest& other) const;
+
+  GCCause::Cause cause() const;
+  uint nworkers() const;
+};
+
+class XDriver : public ConcurrentGCThread {
+private:
+  XMessagePort<XDriverRequest> _gc_cycle_port;
+  XRendezvousPort              _gc_locker_port;
+
+  template <typename T> bool pause();
+
+  void pause_mark_start();
+  void concurrent_mark();
+  bool pause_mark_end();
+  void concurrent_mark_continue();
+  void concurrent_mark_free();
+  void concurrent_process_non_strong_references();
+  void concurrent_reset_relocation_set();
+  void pause_verify();
+  void concurrent_select_relocation_set();
+  void pause_relocate_start();
+  void concurrent_relocate();
+
+  void check_out_of_memory();
+
+  void gc(const XDriverRequest& request);
+
+protected:
+  virtual void run_service();
+  virtual void stop_service();
+
+public:
+  XDriver();
+
+  bool is_busy() const;
+
+  void collect(const XDriverRequest& request);
+};
+
+#endif // SHARE_GC_X_XDRIVER_HPP
diff --git a/src/hotspot/share/gc/x/xErrno.cpp b/src/hotspot/share/gc/x/xErrno.cpp
new file mode 100644
index 00000000000..64951bc47ab
--- /dev/null
+++ b/src/hotspot/share/gc/x/xErrno.cpp
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xErrno.hpp"
+#include "runtime/os.hpp"
+
+#include <errno.h>
+#include <string.h>
+
+XErrno::XErrno() :
+    _error(errno) {}
+
+XErrno::XErrno(int error) :
+    _error(error) {}
+
+XErrno::operator bool() const {
+  return _error != 0;
+}
+
+bool XErrno::operator==(int error) const {
+  return _error == error;
+}
+
+bool XErrno::operator!=(int error) const {
+  return _error != error;
+}
+
+const char* XErrno::to_string() const {
+  return os::strerror(_error);
+}
diff --git a/src/hotspot/share/gc/x/xErrno.hpp b/src/hotspot/share/gc/x/xErrno.hpp
new file mode 100644
index 00000000000..eb72d43da3f
--- /dev/null
+++ b/src/hotspot/share/gc/x/xErrno.hpp
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XERRNO_HPP
+#define SHARE_GC_X_XERRNO_HPP
+
+#include "memory/allocation.hpp"
+
+class XErrno : public StackObj {
+private:
+  const int _error;
+
+public:
+  XErrno();
+  XErrno(int error);
+
+  operator bool() const;
+  bool operator==(int error) const;
+  bool operator!=(int error) const;
+  const char* to_string() const;
+};
+
+#endif // SHARE_GC_X_XERRNO_HPP
diff --git a/src/hotspot/share/gc/x/xForwarding.cpp b/src/hotspot/share/gc/x/xForwarding.cpp
new file mode 100644
index 00000000000..3e8b50d0d64
--- /dev/null
+++ b/src/hotspot/share/gc/x/xForwarding.cpp
@@ -0,0 +1,210 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xForwarding.inline.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xUtils.inline.hpp"
+#include "utilities/align.hpp"
+
+//
+// Reference count states:
+//
+// * If the reference count is zero, it will never change again.
+//
+// * If the reference count is positive, it can be both retained
+//   (increased) and released (decreased).
+//
+// * If the reference count is negative, is can only be released
+//   (increased). A negative reference count means that one or more
+//   threads are waiting for one or more other threads to release
+//   their references.
+//
+// The reference lock is used for waiting until the reference
+// count has become zero (released) or negative one (claimed).
+//
+
+static const XStatCriticalPhase XCriticalPhaseRelocationStall("Relocation Stall");
+
+bool XForwarding::retain_page() {
+  for (;;) {
+    const int32_t ref_count = Atomic::load_acquire(&_ref_count);
+
+    if (ref_count == 0) {
+      // Released
+      return false;
+    }
+
+    if (ref_count < 0) {
+      // Claimed
+      const bool success = wait_page_released();
+      assert(success, "Should always succeed");
+      return false;
+    }
+
+    if (Atomic::cmpxchg(&_ref_count, ref_count, ref_count + 1) == ref_count) {
+      // Retained
+      return true;
+    }
+  }
+}
+
+XPage* XForwarding::claim_page() {
+  for (;;) {
+    const int32_t ref_count = Atomic::load(&_ref_count);
+    assert(ref_count > 0, "Invalid state");
+
+    // Invert reference count
+    if (Atomic::cmpxchg(&_ref_count, ref_count, -ref_count) != ref_count) {
+      continue;
+    }
+
+    // If the previous reference count was 1, then we just changed it to -1,
+    // and we have now claimed the page. Otherwise we wait until it is claimed.
+    if (ref_count != 1) {
+      XLocker<XConditionLock> locker(&_ref_lock);
+      while (Atomic::load_acquire(&_ref_count) != -1) {
+        _ref_lock.wait();
+      }
+    }
+
+    return _page;
+  }
+}
+
+void XForwarding::release_page() {
+  for (;;) {
+    const int32_t ref_count = Atomic::load(&_ref_count);
+    assert(ref_count != 0, "Invalid state");
+
+    if (ref_count > 0) {
+      // Decrement reference count
+      if (Atomic::cmpxchg(&_ref_count, ref_count, ref_count - 1) != ref_count) {
+        continue;
+      }
+
+      // If the previous reference count was 1, then we just decremented
+      // it to 0 and we should signal that the page is now released.
+      if (ref_count == 1) {
+        // Notify released
+        XLocker<XConditionLock> locker(&_ref_lock);
+        _ref_lock.notify_all();
+      }
+    } else {
+      // Increment reference count
+      if (Atomic::cmpxchg(&_ref_count, ref_count, ref_count + 1) != ref_count) {
+        continue;
+      }
+
+      // If the previous reference count was -2 or -1, then we just incremented it
+      // to -1 or 0, and we should signal the that page is now claimed or released.
+      if (ref_count == -2 || ref_count == -1) {
+        // Notify claimed or released
+        XLocker<XConditionLock> locker(&_ref_lock);
+        _ref_lock.notify_all();
+      }
+    }
+
+    return;
+  }
+}
+
+bool XForwarding::wait_page_released() const {
+  if (Atomic::load_acquire(&_ref_count) != 0) {
+    XStatTimer timer(XCriticalPhaseRelocationStall);
+    XLocker<XConditionLock> locker(&_ref_lock);
+    while (Atomic::load_acquire(&_ref_count) != 0) {
+      if (_ref_abort) {
+        return false;
+      }
+
+      _ref_lock.wait();
+    }
+  }
+
+  return true;
+}
+
+XPage* XForwarding::detach_page() {
+  // Wait until released
+  if (Atomic::load_acquire(&_ref_count) != 0) {
+    XLocker<XConditionLock> locker(&_ref_lock);
+    while (Atomic::load_acquire(&_ref_count) != 0) {
+      _ref_lock.wait();
+    }
+  }
+
+  // Detach and return page
+  XPage* const page = _page;
+  _page = NULL;
+  return page;
+}
+
+void XForwarding::abort_page() {
+  XLocker<XConditionLock> locker(&_ref_lock);
+  assert(Atomic::load(&_ref_count) > 0, "Invalid state");
+  assert(!_ref_abort, "Invalid state");
+  _ref_abort = true;
+  _ref_lock.notify_all();
+}
+
+void XForwarding::verify() const {
+  guarantee(_ref_count != 0, "Invalid reference count");
+  guarantee(_page != NULL, "Invalid page");
+
+  uint32_t live_objects = 0;
+  size_t live_bytes = 0;
+
+  for (XForwardingCursor i = 0; i < _entries.length(); i++) {
+    const XForwardingEntry entry = at(&i);
+    if (!entry.populated()) {
+      // Skip empty entries
+      continue;
+    }
+
+    // Check from index
+    guarantee(entry.from_index() < _page->object_max_count(), "Invalid from index");
+
+    // Check for duplicates
+    for (XForwardingCursor j = i + 1; j < _entries.length(); j++) {
+      const XForwardingEntry other = at(&j);
+      if (!other.populated()) {
+        // Skip empty entries
+        continue;
+      }
+
+      guarantee(entry.from_index() != other.from_index(), "Duplicate from");
+      guarantee(entry.to_offset() != other.to_offset(), "Duplicate to");
+    }
+
+    const uintptr_t to_addr = XAddress::good(entry.to_offset());
+    const size_t size = XUtils::object_size(to_addr);
+    const size_t aligned_size = align_up(size, _page->object_alignment());
+    live_bytes += aligned_size;
+    live_objects++;
+  }
+
+  // Verify number of live objects and bytes
+  _page->verify_live(live_objects, live_bytes);
+}
diff --git a/src/hotspot/share/gc/x/xForwarding.hpp b/src/hotspot/share/gc/x/xForwarding.hpp
new file mode 100644
index 00000000000..a6185e23ced
--- /dev/null
+++ b/src/hotspot/share/gc/x/xForwarding.hpp
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XFORWARDING_HPP
+#define SHARE_GC_X_XFORWARDING_HPP
+
+#include "gc/x/xAttachedArray.hpp"
+#include "gc/x/xForwardingEntry.hpp"
+#include "gc/x/xLock.hpp"
+#include "gc/x/xVirtualMemory.hpp"
+
+class ObjectClosure;
+class VMStructs;
+class XForwardingAllocator;
+class XPage;
+
+typedef size_t XForwardingCursor;
+
+class XForwarding {
+  friend class ::VMStructs;
+  friend class XForwardingTest;
+
+private:
+  typedef XAttachedArray<XForwarding, XForwardingEntry> AttachedArray;
+
+  const XVirtualMemory   _virtual;
+  const size_t           _object_alignment_shift;
+  const AttachedArray    _entries;
+  XPage*                 _page;
+  mutable XConditionLock _ref_lock;
+  volatile int32_t       _ref_count;
+  bool                   _ref_abort;
+  bool                   _in_place;
+
+  XForwardingEntry* entries() const;
+  XForwardingEntry at(XForwardingCursor* cursor) const;
+  XForwardingEntry first(uintptr_t from_index, XForwardingCursor* cursor) const;
+  XForwardingEntry next(XForwardingCursor* cursor) const;
+
+  XForwarding(XPage* page, size_t nentries);
+
+public:
+  static uint32_t nentries(const XPage* page);
+  static XForwarding* alloc(XForwardingAllocator* allocator, XPage* page);
+
+  uint8_t type() const;
+  uintptr_t start() const;
+  size_t size() const;
+  size_t object_alignment_shift() const;
+  void object_iterate(ObjectClosure *cl);
+
+  bool retain_page();
+  XPage* claim_page();
+  void release_page();
+  bool wait_page_released() const;
+  XPage* detach_page();
+  void abort_page();
+
+  void set_in_place();
+  bool in_place() const;
+
+  XForwardingEntry find(uintptr_t from_index, XForwardingCursor* cursor) const;
+  uintptr_t insert(uintptr_t from_index, uintptr_t to_offset, XForwardingCursor* cursor);
+
+  void verify() const;
+};
+
+#endif // SHARE_GC_X_XFORWARDING_HPP
diff --git a/src/hotspot/share/gc/x/xForwarding.inline.hpp b/src/hotspot/share/gc/x/xForwarding.inline.hpp
new file mode 100644
index 00000000000..257109f3de9
--- /dev/null
+++ b/src/hotspot/share/gc/x/xForwarding.inline.hpp
@@ -0,0 +1,163 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XFORWARDING_INLINE_HPP
+#define SHARE_GC_X_XFORWARDING_INLINE_HPP
+
+#include "gc/x/xForwarding.hpp"
+
+#include "gc/x/xAttachedArray.inline.hpp"
+#include "gc/x/xForwardingAllocator.inline.hpp"
+#include "gc/x/xHash.inline.hpp"
+#include "gc/x/xHeap.hpp"
+#include "gc/x/xLock.inline.hpp"
+#include "gc/x/xPage.inline.hpp"
+#include "gc/x/xVirtualMemory.inline.hpp"
+#include "runtime/atomic.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+inline uint32_t XForwarding::nentries(const XPage* page) {
+  // The number returned by the function is used to size the hash table of
+  // forwarding entries for this page. This hash table uses linear probing.
+  // The size of the table must be a power of two to allow for quick and
+  // inexpensive indexing/masking. The table is also sized to have a load
+  // factor of 50%, i.e. sized to have double the number of entries actually
+  // inserted, to allow for good lookup/insert performance.
+  return round_up_power_of_2(page->live_objects() * 2);
+}
+
+inline XForwarding* XForwarding::alloc(XForwardingAllocator* allocator, XPage* page) {
+  const size_t nentries = XForwarding::nentries(page);
+  void* const addr = AttachedArray::alloc(allocator, nentries);
+  return ::new (addr) XForwarding(page, nentries);
+}
+
+inline XForwarding::XForwarding(XPage* page, size_t nentries) :
+    _virtual(page->virtual_memory()),
+    _object_alignment_shift(page->object_alignment_shift()),
+    _entries(nentries),
+    _page(page),
+    _ref_lock(),
+    _ref_count(1),
+    _ref_abort(false),
+    _in_place(false) {}
+
+inline uint8_t XForwarding::type() const {
+  return _page->type();
+}
+
+inline uintptr_t XForwarding::start() const {
+  return _virtual.start();
+}
+
+inline size_t XForwarding::size() const {
+  return _virtual.size();
+}
+
+inline size_t XForwarding::object_alignment_shift() const {
+  return _object_alignment_shift;
+}
+
+inline void XForwarding::object_iterate(ObjectClosure *cl) {
+  return _page->object_iterate(cl);
+}
+
+inline void XForwarding::set_in_place() {
+  _in_place = true;
+}
+
+inline bool XForwarding::in_place() const {
+  return _in_place;
+}
+
+inline XForwardingEntry* XForwarding::entries() const {
+  return _entries(this);
+}
+
+inline XForwardingEntry XForwarding::at(XForwardingCursor* cursor) const {
+  // Load acquire for correctness with regards to
+  // accesses to the contents of the forwarded object.
+  return Atomic::load_acquire(entries() + *cursor);
+}
+
+inline XForwardingEntry XForwarding::first(uintptr_t from_index, XForwardingCursor* cursor) const {
+  const size_t mask = _entries.length() - 1;
+  const size_t hash = XHash::uint32_to_uint32((uint32_t)from_index);
+  *cursor = hash & mask;
+  return at(cursor);
+}
+
+inline XForwardingEntry XForwarding::next(XForwardingCursor* cursor) const {
+  const size_t mask = _entries.length() - 1;
+  *cursor = (*cursor + 1) & mask;
+  return at(cursor);
+}
+
+inline XForwardingEntry XForwarding::find(uintptr_t from_index, XForwardingCursor* cursor) const {
+  // Reading entries in the table races with the atomic CAS done for
+  // insertion into the table. This is safe because each entry is at
+  // most updated once (from zero to something else).
+  XForwardingEntry entry = first(from_index, cursor);
+  while (entry.populated()) {
+    if (entry.from_index() == from_index) {
+      // Match found, return matching entry
+      return entry;
+    }
+
+    entry = next(cursor);
+  }
+
+  // Match not found, return empty entry
+  return entry;
+}
+
+inline uintptr_t XForwarding::insert(uintptr_t from_index, uintptr_t to_offset, XForwardingCursor* cursor) {
+  const XForwardingEntry new_entry(from_index, to_offset);
+  const XForwardingEntry old_entry; // Empty
+
+  // Make sure that object copy is finished
+  // before forwarding table installation
+  OrderAccess::release();
+
+  for (;;) {
+    const XForwardingEntry prev_entry = Atomic::cmpxchg(entries() + *cursor, old_entry, new_entry, memory_order_relaxed);
+    if (!prev_entry.populated()) {
+      // Success
+      return to_offset;
+    }
+
+    // Find next empty or matching entry
+    XForwardingEntry entry = at(cursor);
+    while (entry.populated()) {
+      if (entry.from_index() == from_index) {
+        // Match found, return already inserted address
+        return entry.to_offset();
+      }
+
+      entry = next(cursor);
+    }
+  }
+}
+
+#endif // SHARE_GC_X_XFORWARDING_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xForwardingAllocator.cpp b/src/hotspot/share/gc/x/xForwardingAllocator.cpp
new file mode 100644
index 00000000000..fddff50e88f
--- /dev/null
+++ b/src/hotspot/share/gc/x/xForwardingAllocator.cpp
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xForwardingAllocator.hpp"
+#include "memory/allocation.inline.hpp"
+
+XForwardingAllocator::XForwardingAllocator() :
+    _start(NULL),
+    _end(NULL),
+    _top(NULL) {}
+
+XForwardingAllocator::~XForwardingAllocator() {
+  FREE_C_HEAP_ARRAY(char, _start);
+}
+
+void XForwardingAllocator::reset(size_t size) {
+  _start = _top = REALLOC_C_HEAP_ARRAY(char, _start, size, mtGC);
+  _end = _start + size;
+}
diff --git a/src/hotspot/share/gc/x/xForwardingAllocator.hpp b/src/hotspot/share/gc/x/xForwardingAllocator.hpp
new file mode 100644
index 00000000000..75495944e8a
--- /dev/null
+++ b/src/hotspot/share/gc/x/xForwardingAllocator.hpp
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XFORWARDINGALLOCATOR_HPP
+#define SHARE_GC_X_XFORWARDINGALLOCATOR_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+class XForwardingAllocator {
+private:
+  char* _start;
+  char* _end;
+  char* _top;
+
+public:
+  XForwardingAllocator();
+  ~XForwardingAllocator();
+
+  void reset(size_t size);
+  size_t size() const;
+  bool is_full() const;
+
+  void* alloc(size_t size);
+};
+
+#endif // SHARE_GC_X_XFORWARDINGALLOCATOR_HPP
diff --git a/src/hotspot/share/gc/x/xForwardingAllocator.inline.hpp b/src/hotspot/share/gc/x/xForwardingAllocator.inline.hpp
new file mode 100644
index 00000000000..ebbf02dbd8f
--- /dev/null
+++ b/src/hotspot/share/gc/x/xForwardingAllocator.inline.hpp
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XFORWARDINGALLOCATOR_INLINE_HPP
+#define SHARE_GC_X_XFORWARDINGALLOCATOR_INLINE_HPP
+
+#include "gc/x/xForwardingAllocator.hpp"
+
+#include "runtime/atomic.hpp"
+#include "utilities/debug.hpp"
+
+inline size_t XForwardingAllocator::size() const {
+  return _end - _start;
+}
+
+inline bool XForwardingAllocator::is_full() const {
+  return _top == _end;
+}
+
+inline void* XForwardingAllocator::alloc(size_t size) {
+  char* const addr = Atomic::fetch_and_add(&_top, size);
+  assert(addr + size <= _end, "Allocation should never fail");
+  return addr;
+}
+
+#endif // SHARE_GC_X_XFORWARDINGALLOCATOR_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xForwardingEntry.hpp b/src/hotspot/share/gc/x/xForwardingEntry.hpp
new file mode 100644
index 00000000000..3f8846abbaa
--- /dev/null
+++ b/src/hotspot/share/gc/x/xForwardingEntry.hpp
@@ -0,0 +1,102 @@
+/*
+ * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XFORWARDINGENTRY_HPP
+#define SHARE_GC_X_XFORWARDINGENTRY_HPP
+
+#include "gc/x/xBitField.hpp"
+#include "memory/allocation.hpp"
+#include "metaprogramming/primitiveConversions.hpp"
+
+#include <type_traits>
+
+class VMStructs;
+
+//
+// Forwarding entry layout
+// -----------------------
+//
+//   6                  4 4
+//   3                  6 5                                                1 0
+//  +--------------------+--------------------------------------------------+-+
+//  |11111111 11111111 11|111111 11111111 11111111 11111111 11111111 1111111|1|
+//  +--------------------+--------------------------------------------------+-+
+//  |                    |                                                  |
+//  |                    |                      0-0 Populated Flag (1-bits) *
+//  |                    |
+//  |                    * 45-1 To Object Offset (45-bits)
+//  |
+//  * 63-46 From Object Index (18-bits)
+//
+
+class XForwardingEntry {
+  friend struct PrimitiveConversions::Translate<XForwardingEntry>;
+  friend class ::VMStructs;
+
+private:
+  typedef XBitField<uint64_t, bool,   0,   1> field_populated;
+  typedef XBitField<uint64_t, size_t, 1,  45> field_to_offset;
+  typedef XBitField<uint64_t, size_t, 46, 18> field_from_index;
+
+  uint64_t _entry;
+
+public:
+  XForwardingEntry() :
+      _entry(0) {}
+
+  XForwardingEntry(size_t from_index, size_t to_offset) :
+      _entry(field_populated::encode(true) |
+             field_to_offset::encode(to_offset) |
+             field_from_index::encode(from_index)) {}
+
+  bool populated() const {
+    return field_populated::decode(_entry);
+  }
+
+  size_t to_offset() const {
+    return field_to_offset::decode(_entry);
+  }
+
+  size_t from_index() const {
+    return field_from_index::decode(_entry);
+  }
+};
+
+// Needed to allow atomic operations on XForwardingEntry
+template <>
+struct PrimitiveConversions::Translate<XForwardingEntry> : public std::true_type {
+  typedef XForwardingEntry Value;
+  typedef uint64_t                  Decayed;
+
+  static Decayed decay(Value v) {
+    return v._entry;
+  }
+
+  static Value recover(Decayed d) {
+    XForwardingEntry entry;
+    entry._entry = d;
+    return entry;
+  }
+};
+
+#endif // SHARE_GC_X_XFORWARDINGENTRY_HPP
diff --git a/src/hotspot/share/gc/x/xForwardingTable.hpp b/src/hotspot/share/gc/x/xForwardingTable.hpp
new file mode 100644
index 00000000000..1f110292be5
--- /dev/null
+++ b/src/hotspot/share/gc/x/xForwardingTable.hpp
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XFORWARDINGTABLE_HPP
+#define SHARE_GC_X_XFORWARDINGTABLE_HPP
+
+#include "gc/x/xGranuleMap.hpp"
+
+class VMStructs;
+class XForwarding;
+
+class XForwardingTable {
+  friend class ::VMStructs;
+
+private:
+  XGranuleMap<XForwarding*> _map;
+
+public:
+  XForwardingTable();
+
+  XForwarding* get(uintptr_t addr) const;
+
+  void insert(XForwarding* forwarding);
+  void remove(XForwarding* forwarding);
+};
+
+#endif // SHARE_GC_X_XFORWARDINGTABLE_HPP
diff --git a/src/hotspot/share/gc/x/xForwardingTable.inline.hpp b/src/hotspot/share/gc/x/xForwardingTable.inline.hpp
new file mode 100644
index 00000000000..3ea30d383ec
--- /dev/null
+++ b/src/hotspot/share/gc/x/xForwardingTable.inline.hpp
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XFORWARDINGTABLE_INLINE_HPP
+#define SHARE_GC_X_XFORWARDINGTABLE_INLINE_HPP
+
+#include "gc/x/xForwardingTable.hpp"
+
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xForwarding.inline.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xGranuleMap.inline.hpp"
+#include "utilities/debug.hpp"
+
+inline XForwardingTable::XForwardingTable() :
+    _map(XAddressOffsetMax) {}
+
+inline XForwarding* XForwardingTable::get(uintptr_t addr) const {
+  assert(!XAddress::is_null(addr), "Invalid address");
+  return _map.get(XAddress::offset(addr));
+}
+
+inline void XForwardingTable::insert(XForwarding* forwarding) {
+  const uintptr_t offset = forwarding->start();
+  const size_t size = forwarding->size();
+
+  assert(_map.get(offset) == NULL, "Invalid entry");
+  _map.put(offset, size, forwarding);
+}
+
+inline void XForwardingTable::remove(XForwarding* forwarding) {
+  const uintptr_t offset = forwarding->start();
+  const size_t size = forwarding->size();
+
+  assert(_map.get(offset) == forwarding, "Invalid entry");
+  _map.put(offset, size, NULL);
+}
+
+#endif // SHARE_GC_X_XFORWARDINGTABLE_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xFuture.hpp b/src/hotspot/share/gc/x/xFuture.hpp
new file mode 100644
index 00000000000..931f4b58f12
--- /dev/null
+++ b/src/hotspot/share/gc/x/xFuture.hpp
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XFUTURE_HPP
+#define SHARE_GC_X_XFUTURE_HPP
+
+#include "memory/allocation.hpp"
+#include "runtime/semaphore.hpp"
+
+template <typename T>
+class XFuture {
+private:
+  Semaphore _sema;
+  T         _value;
+
+public:
+  XFuture();
+
+  void set(T value);
+  T get();
+};
+
+#endif // SHARE_GC_X_XFUTURE_HPP
diff --git a/src/hotspot/share/gc/x/xFuture.inline.hpp b/src/hotspot/share/gc/x/xFuture.inline.hpp
new file mode 100644
index 00000000000..d3dba3b7151
--- /dev/null
+++ b/src/hotspot/share/gc/x/xFuture.inline.hpp
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XFUTURE_INLINE_HPP
+#define SHARE_GC_X_XFUTURE_INLINE_HPP
+
+#include "gc/x/xFuture.hpp"
+
+#include "runtime/javaThread.hpp"
+#include "runtime/semaphore.inline.hpp"
+
+template <typename T>
+inline XFuture<T>::XFuture() :
+    _value() {}
+
+template <typename T>
+inline void XFuture<T>::set(T value) {
+  // Set value
+  _value = value;
+
+  // Notify waiter
+  _sema.signal();
+}
+
+template <typename T>
+inline T XFuture<T>::get() {
+  // Wait for notification
+  Thread* const thread = Thread::current();
+  if (thread->is_Java_thread()) {
+    _sema.wait_with_safepoint_check(JavaThread::cast(thread));
+  } else {
+    _sema.wait();
+  }
+
+  // Return value
+  return _value;
+}
+
+#endif // SHARE_GC_X_XFUTURE_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xGlobals.cpp b/src/hotspot/share/gc/x/xGlobals.cpp
new file mode 100644
index 00000000000..b247565bc01
--- /dev/null
+++ b/src/hotspot/share/gc/x/xGlobals.cpp
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xGlobals.hpp"
+
+uint32_t   XGlobalPhase                = XPhaseRelocate;
+uint32_t   XGlobalSeqNum               = 1;
+
+size_t     XPageSizeMediumShift;
+size_t     XPageSizeMedium;
+
+size_t     XObjectSizeLimitMedium;
+
+const int& XObjectAlignmentSmallShift  = LogMinObjAlignmentInBytes;
+int        XObjectAlignmentMediumShift;
+
+const int& XObjectAlignmentSmall       = MinObjAlignmentInBytes;
+int        XObjectAlignmentMedium;
+
+uintptr_t  XAddressGoodMask;
+uintptr_t  XAddressBadMask;
+uintptr_t  XAddressWeakBadMask;
+
+static uint32_t* XAddressCalculateBadMaskHighOrderBitsAddr() {
+  const uintptr_t addr = reinterpret_cast<uintptr_t>(&XAddressBadMask);
+  return reinterpret_cast<uint32_t*>(addr + XAddressBadMaskHighOrderBitsOffset);
+}
+
+uint32_t*  XAddressBadMaskHighOrderBitsAddr = XAddressCalculateBadMaskHighOrderBitsAddr();
+
+size_t     XAddressOffsetBits;
+uintptr_t  XAddressOffsetMask;
+size_t     XAddressOffsetMax;
+
+size_t     XAddressMetadataShift;
+uintptr_t  XAddressMetadataMask;
+
+uintptr_t  XAddressMetadataMarked;
+uintptr_t  XAddressMetadataMarked0;
+uintptr_t  XAddressMetadataMarked1;
+uintptr_t  XAddressMetadataRemapped;
+uintptr_t  XAddressMetadataFinalizable;
+
+const char* XGlobalPhaseToString() {
+  switch (XGlobalPhase) {
+  case XPhaseMark:
+    return "Mark";
+
+  case XPhaseMarkCompleted:
+    return "MarkCompleted";
+
+  case XPhaseRelocate:
+    return "Relocate";
+
+  default:
+    return "Unknown";
+  }
+}
diff --git a/src/hotspot/share/gc/x/xGlobals.hpp b/src/hotspot/share/gc/x/xGlobals.hpp
new file mode 100644
index 00000000000..662a502a79f
--- /dev/null
+++ b/src/hotspot/share/gc/x/xGlobals.hpp
@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XGLOBALS_HPP
+#define SHARE_GC_X_XGLOBALS_HPP
+
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/macros.hpp"
+#include CPU_HEADER(gc/x/xGlobals)
+
+// Collector name
+const char* const XName                         = "The Z Garbage Collector";
+
+// Global phase state
+extern uint32_t   XGlobalPhase;
+const uint32_t    XPhaseMark                    = 0;
+const uint32_t    XPhaseMarkCompleted           = 1;
+const uint32_t    XPhaseRelocate                = 2;
+const char*       XGlobalPhaseToString();
+
+// Global sequence number
+extern uint32_t   XGlobalSeqNum;
+
+// Granule shift/size
+const size_t      XGranuleSizeShift             = 21; // 2MB
+const size_t      XGranuleSize                  = (size_t)1 << XGranuleSizeShift;
+
+// Number of heap views
+const size_t      XHeapViews                    = XPlatformHeapViews;
+
+// Virtual memory to physical memory ratio
+const size_t      XVirtualToPhysicalRatio       = 16; // 16:1
+
+// Page types
+const uint8_t     XPageTypeSmall                = 0;
+const uint8_t     XPageTypeMedium               = 1;
+const uint8_t     XPageTypeLarge                = 2;
+
+// Page size shifts
+const size_t      XPageSizeSmallShift           = XGranuleSizeShift;
+extern size_t     XPageSizeMediumShift;
+
+// Page sizes
+const size_t      XPageSizeSmall                = (size_t)1 << XPageSizeSmallShift;
+extern size_t     XPageSizeMedium;
+
+// Object size limits
+const size_t      XObjectSizeLimitSmall         = XPageSizeSmall / 8; // 12.5% max waste
+extern size_t     XObjectSizeLimitMedium;
+
+// Object alignment shifts
+extern const int& XObjectAlignmentSmallShift;
+extern int        XObjectAlignmentMediumShift;
+const int         XObjectAlignmentLargeShift    = XGranuleSizeShift;
+
+// Object alignments
+extern const int& XObjectAlignmentSmall;
+extern int        XObjectAlignmentMedium;
+const int         XObjectAlignmentLarge         = 1 << XObjectAlignmentLargeShift;
+
+//
+// Good/Bad mask states
+// --------------------
+//
+//                 GoodMask         BadMask          WeakGoodMask     WeakBadMask
+//                 --------------------------------------------------------------
+//  Marked0        001              110              101              010
+//  Marked1        010              101              110              001
+//  Remapped       100              011              100              011
+//
+
+// Good/bad masks
+extern uintptr_t  XAddressGoodMask;
+extern uintptr_t  XAddressBadMask;
+extern uintptr_t  XAddressWeakBadMask;
+
+// The bad mask is 64 bit. Its high order 32 bits contain all possible value combinations
+// that this mask will have. Therefore, the memory where the 32 high order bits are stored,
+// can be used as a 32 bit GC epoch counter, that has a different bit pattern every time
+// the bad mask is flipped. This provides a pointer to said 32 bits.
+extern uint32_t*  XAddressBadMaskHighOrderBitsAddr;
+const int         XAddressBadMaskHighOrderBitsOffset = LITTLE_ENDIAN_ONLY(4) BIG_ENDIAN_ONLY(0);
+
+// Pointer part of address
+extern size_t     XAddressOffsetBits;
+const  size_t     XAddressOffsetShift           = 0;
+extern uintptr_t  XAddressOffsetMask;
+extern size_t     XAddressOffsetMax;
+
+// Metadata part of address
+const size_t      XAddressMetadataBits          = 4;
+extern size_t     XAddressMetadataShift;
+extern uintptr_t  XAddressMetadataMask;
+
+// Metadata types
+extern uintptr_t  XAddressMetadataMarked;
+extern uintptr_t  XAddressMetadataMarked0;
+extern uintptr_t  XAddressMetadataMarked1;
+extern uintptr_t  XAddressMetadataRemapped;
+extern uintptr_t  XAddressMetadataFinalizable;
+
+// Cache line size
+const size_t      XCacheLineSize                = XPlatformCacheLineSize;
+#define           XCACHE_ALIGNED                ATTRIBUTE_ALIGNED(XCacheLineSize)
+
+// Mark stack space
+extern uintptr_t  XMarkStackSpaceStart;
+const size_t      XMarkStackSpaceExpandSize     = (size_t)1 << 25; // 32M
+
+// Mark stack and magazine sizes
+const size_t      XMarkStackSizeShift           = 11; // 2K
+const size_t      XMarkStackSize                = (size_t)1 << XMarkStackSizeShift;
+const size_t      XMarkStackHeaderSize          = (size_t)1 << 4; // 16B
+const size_t      XMarkStackSlots               = (XMarkStackSize - XMarkStackHeaderSize) / sizeof(uintptr_t);
+const size_t      XMarkStackMagazineSize        = (size_t)1 << 15; // 32K
+const size_t      XMarkStackMagazineSlots       = (XMarkStackMagazineSize / XMarkStackSize) - 1;
+
+// Mark stripe size
+const size_t      XMarkStripeShift              = XGranuleSizeShift;
+
+// Max number of mark stripes
+const size_t      XMarkStripesMax               = 16; // Must be a power of two
+
+// Mark cache size
+const size_t      XMarkCacheSize                = 1024; // Must be a power of two
+
+// Partial array minimum size
+const size_t      XMarkPartialArrayMinSizeShift = 12; // 4K
+const size_t      XMarkPartialArrayMinSize      = (size_t)1 << XMarkPartialArrayMinSizeShift;
+
+// Max number of proactive/terminate flush attempts
+const size_t      XMarkProactiveFlushMax        = 10;
+const size_t      XMarkTerminateFlushMax        = 3;
+
+// Try complete mark timeout
+const uint64_t    XMarkCompleteTimeout          = 200; // us
+
+#endif // SHARE_GC_X_XGLOBALS_HPP
diff --git a/src/hotspot/share/gc/x/xGranuleMap.hpp b/src/hotspot/share/gc/x/xGranuleMap.hpp
new file mode 100644
index 00000000000..a9447e1469c
--- /dev/null
+++ b/src/hotspot/share/gc/x/xGranuleMap.hpp
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XGRANULEMAP_HPP
+#define SHARE_GC_X_XGRANULEMAP_HPP
+
+#include "gc/x/xArray.hpp"
+#include "memory/allocation.hpp"
+
+class VMStructs;
+
+template <typename T>
+class XGranuleMap {
+  friend class ::VMStructs;
+  template <typename> friend class XGranuleMapIterator;
+
+private:
+  const size_t _size;
+  T* const     _map;
+
+  size_t index_for_offset(uintptr_t offset) const;
+
+public:
+  XGranuleMap(size_t max_offset);
+  ~XGranuleMap();
+
+  T get(uintptr_t offset) const;
+  void put(uintptr_t offset, T value);
+  void put(uintptr_t offset, size_t size, T value);
+
+  T get_acquire(uintptr_t offset) const;
+  void release_put(uintptr_t offset, T value);
+};
+
+template <typename T>
+class XGranuleMapIterator : public XArrayIteratorImpl<T, false /* Parallel */> {
+public:
+  XGranuleMapIterator(const XGranuleMap<T>* granule_map);
+};
+
+#endif // SHARE_GC_X_XGRANULEMAP_HPP
diff --git a/src/hotspot/share/gc/x/xGranuleMap.inline.hpp b/src/hotspot/share/gc/x/xGranuleMap.inline.hpp
new file mode 100644
index 00000000000..95ef5ee2b2d
--- /dev/null
+++ b/src/hotspot/share/gc/x/xGranuleMap.inline.hpp
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XGRANULEMAP_INLINE_HPP
+#define SHARE_GC_X_XGRANULEMAP_INLINE_HPP
+
+#include "gc/x/xGranuleMap.hpp"
+
+#include "gc/x/xArray.inline.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "memory/allocation.inline.hpp"
+#include "runtime/atomic.hpp"
+#include "utilities/align.hpp"
+#include "utilities/debug.hpp"
+
+template <typename T>
+inline XGranuleMap<T>::XGranuleMap(size_t max_offset) :
+    _size(max_offset >> XGranuleSizeShift),
+    _map(MmapArrayAllocator<T>::allocate(_size, mtGC)) {
+  assert(is_aligned(max_offset, XGranuleSize), "Misaligned");
+}
+
+template <typename T>
+inline XGranuleMap<T>::~XGranuleMap() {
+  MmapArrayAllocator<T>::free(_map, _size);
+}
+
+template <typename T>
+inline size_t XGranuleMap<T>::index_for_offset(uintptr_t offset) const {
+  const size_t index = offset >> XGranuleSizeShift;
+  assert(index < _size, "Invalid index");
+  return index;
+}
+
+template <typename T>
+inline T XGranuleMap<T>::get(uintptr_t offset) const {
+  const size_t index = index_for_offset(offset);
+  return _map[index];
+}
+
+template <typename T>
+inline void XGranuleMap<T>::put(uintptr_t offset, T value) {
+  const size_t index = index_for_offset(offset);
+  _map[index] = value;
+}
+
+template <typename T>
+inline void XGranuleMap<T>::put(uintptr_t offset, size_t size, T value) {
+  assert(is_aligned(size, XGranuleSize), "Misaligned");
+
+  const size_t start_index = index_for_offset(offset);
+  const size_t end_index = start_index + (size >> XGranuleSizeShift);
+  for (size_t index = start_index; index < end_index; index++) {
+    _map[index] = value;
+  }
+}
+
+template <typename T>
+inline T XGranuleMap<T>::get_acquire(uintptr_t offset) const {
+  const size_t index = index_for_offset(offset);
+  return Atomic::load_acquire(_map + index);
+}
+
+template <typename T>
+inline void XGranuleMap<T>::release_put(uintptr_t offset, T value) {
+  const size_t index = index_for_offset(offset);
+  Atomic::release_store(_map + index, value);
+}
+
+template <typename T>
+inline XGranuleMapIterator<T>::XGranuleMapIterator(const XGranuleMap<T>* granule_map) :
+    XArrayIteratorImpl<T, false /* Parallel */>(granule_map->_map, granule_map->_size) {}
+
+#endif // SHARE_GC_X_XGRANULEMAP_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xHash.hpp b/src/hotspot/share/gc/x/xHash.hpp
new file mode 100644
index 00000000000..253f4d231c1
--- /dev/null
+++ b/src/hotspot/share/gc/x/xHash.hpp
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XHASH_HPP
+#define SHARE_GC_X_XHASH_HPP
+
+#include "memory/allStatic.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class XHash : public AllStatic {
+public:
+  static uint32_t uint32_to_uint32(uint32_t key);
+  static uint32_t address_to_uint32(uintptr_t key);
+};
+
+#endif // SHARE_GC_X_XHASH_HPP
diff --git a/src/hotspot/share/gc/x/xHash.inline.hpp b/src/hotspot/share/gc/x/xHash.inline.hpp
new file mode 100644
index 00000000000..5ff5f540821
--- /dev/null
+++ b/src/hotspot/share/gc/x/xHash.inline.hpp
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * This file is available under and governed by the GNU General Public
+ * License version 2 only, as published by the Free Software Foundation.
+ * However, the following notice accompanied the original version of this
+ * file:
+ *
+ *  (C) 2009 by Remo Dentato (rdentato@gmail.com)
+ *
+ *
+ * Redistribution and use in source and binary forms, with or without modification,
+ * are permitted provided that the following conditions are met:
+ *
+ *     * Redistributions of source code must retain the above copyright notice,
+ *       this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above copyright notice,
+ *       this list of conditions and the following disclaimer in the documentation
+ *       and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * http://opensource.org/licenses/bsd-license.php
+ */
+
+#ifndef SHARE_GC_X_XHASH_INLINE_HPP
+#define SHARE_GC_X_XHASH_INLINE_HPP
+
+#include "gc/x/xHash.hpp"
+
+#include "gc/x/xAddress.inline.hpp"
+
+inline uint32_t XHash::uint32_to_uint32(uint32_t key) {
+  key = ~key + (key << 15);
+  key = key ^ (key >> 12);
+  key = key + (key << 2);
+  key = key ^ (key >> 4);
+  key = key * 2057;
+  key = key ^ (key >> 16);
+  return key;
+}
+
+inline uint32_t XHash::address_to_uint32(uintptr_t key) {
+  return uint32_to_uint32((uint32_t)(key >> 3));
+}
+
+#endif // SHARE_GC_X_XHASH_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xHeap.cpp b/src/hotspot/share/gc/x/xHeap.cpp
new file mode 100644
index 00000000000..21c4d447c9a
--- /dev/null
+++ b/src/hotspot/share/gc/x/xHeap.cpp
@@ -0,0 +1,536 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "classfile/classLoaderDataGraph.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/shared/locationPrinter.hpp"
+#include "gc/shared/tlab_globals.hpp"
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xArray.inline.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xHeap.inline.hpp"
+#include "gc/x/xHeapIterator.hpp"
+#include "gc/x/xHeuristics.hpp"
+#include "gc/x/xMark.inline.hpp"
+#include "gc/x/xPage.inline.hpp"
+#include "gc/x/xPageTable.inline.hpp"
+#include "gc/x/xRelocationSet.inline.hpp"
+#include "gc/x/xRelocationSetSelector.inline.hpp"
+#include "gc/x/xResurrection.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xThread.inline.hpp"
+#include "gc/x/xVerify.hpp"
+#include "gc/x/xWorkers.hpp"
+#include "logging/log.hpp"
+#include "memory/iterator.hpp"
+#include "memory/metaspaceUtils.hpp"
+#include "memory/resourceArea.hpp"
+#include "prims/jvmtiTagMap.hpp"
+#include "runtime/handshake.hpp"
+#include "runtime/javaThread.hpp"
+#include "runtime/safepoint.hpp"
+#include "utilities/debug.hpp"
+
+static const XStatCounter XCounterUndoPageAllocation("Memory", "Undo Page Allocation", XStatUnitOpsPerSecond);
+static const XStatCounter XCounterOutOfMemory("Memory", "Out Of Memory", XStatUnitOpsPerSecond);
+
+XHeap* XHeap::_heap = NULL;
+
+XHeap::XHeap() :
+    _workers(),
+    _object_allocator(),
+    _page_allocator(&_workers, MinHeapSize, InitialHeapSize, MaxHeapSize),
+    _page_table(),
+    _forwarding_table(),
+    _mark(&_workers, &_page_table),
+    _reference_processor(&_workers),
+    _weak_roots_processor(&_workers),
+    _relocate(&_workers),
+    _relocation_set(&_workers),
+    _unload(&_workers),
+    _serviceability(min_capacity(), max_capacity()) {
+  // Install global heap instance
+  assert(_heap == NULL, "Already initialized");
+  _heap = this;
+
+  // Update statistics
+  XStatHeap::set_at_initialize(_page_allocator.stats());
+}
+
+bool XHeap::is_initialized() const {
+  return _page_allocator.is_initialized() && _mark.is_initialized();
+}
+
+size_t XHeap::min_capacity() const {
+  return _page_allocator.min_capacity();
+}
+
+size_t XHeap::max_capacity() const {
+  return _page_allocator.max_capacity();
+}
+
+size_t XHeap::soft_max_capacity() const {
+  return _page_allocator.soft_max_capacity();
+}
+
+size_t XHeap::capacity() const {
+  return _page_allocator.capacity();
+}
+
+size_t XHeap::used() const {
+  return _page_allocator.used();
+}
+
+size_t XHeap::unused() const {
+  return _page_allocator.unused();
+}
+
+size_t XHeap::tlab_capacity() const {
+  return capacity();
+}
+
+size_t XHeap::tlab_used() const {
+  return _object_allocator.used();
+}
+
+size_t XHeap::max_tlab_size() const {
+  return XObjectSizeLimitSmall;
+}
+
+size_t XHeap::unsafe_max_tlab_alloc() const {
+  size_t size = _object_allocator.remaining();
+
+  if (size < MinTLABSize) {
+    // The remaining space in the allocator is not enough to
+    // fit the smallest possible TLAB. This means that the next
+    // TLAB allocation will force the allocator to get a new
+    // backing page anyway, which in turn means that we can then
+    // fit the largest possible TLAB.
+    size = max_tlab_size();
+  }
+
+  return MIN2(size, max_tlab_size());
+}
+
+bool XHeap::is_in(uintptr_t addr) const {
+  // An address is considered to be "in the heap" if it points into
+  // the allocated part of a page, regardless of which heap view is
+  // used. Note that an address with the finalizable metadata bit set
+  // is not pointing into a heap view, and therefore not considered
+  // to be "in the heap".
+
+  if (XAddress::is_in(addr)) {
+    const XPage* const page = _page_table.get(addr);
+    if (page != NULL) {
+      return page->is_in(addr);
+    }
+  }
+
+  return false;
+}
+
+uint XHeap::active_workers() const {
+  return _workers.active_workers();
+}
+
+void XHeap::set_active_workers(uint nworkers) {
+  _workers.set_active_workers(nworkers);
+}
+
+void XHeap::threads_do(ThreadClosure* tc) const {
+  _page_allocator.threads_do(tc);
+  _workers.threads_do(tc);
+}
+
+void XHeap::out_of_memory() {
+  ResourceMark rm;
+
+  XStatInc(XCounterOutOfMemory);
+  log_info(gc)("Out Of Memory (%s)", Thread::current()->name());
+}
+
+XPage* XHeap::alloc_page(uint8_t type, size_t size, XAllocationFlags flags) {
+  XPage* const page = _page_allocator.alloc_page(type, size, flags);
+  if (page != NULL) {
+    // Insert page table entry
+    _page_table.insert(page);
+  }
+
+  return page;
+}
+
+void XHeap::undo_alloc_page(XPage* page) {
+  assert(page->is_allocating(), "Invalid page state");
+
+  XStatInc(XCounterUndoPageAllocation);
+  log_trace(gc)("Undo page allocation, thread: " PTR_FORMAT " (%s), page: " PTR_FORMAT ", size: " SIZE_FORMAT,
+                XThread::id(), XThread::name(), p2i(page), page->size());
+
+  free_page(page, false /* reclaimed */);
+}
+
+void XHeap::free_page(XPage* page, bool reclaimed) {
+  // Remove page table entry
+  _page_table.remove(page);
+
+  // Free page
+  _page_allocator.free_page(page, reclaimed);
+}
+
+void XHeap::free_pages(const XArray<XPage*>* pages, bool reclaimed) {
+  // Remove page table entries
+  XArrayIterator<XPage*> iter(pages);
+  for (XPage* page; iter.next(&page);) {
+    _page_table.remove(page);
+  }
+
+  // Free pages
+  _page_allocator.free_pages(pages, reclaimed);
+}
+
+void XHeap::flip_to_marked() {
+  XVerifyViewsFlip flip(&_page_allocator);
+  XAddress::flip_to_marked();
+}
+
+void XHeap::flip_to_remapped() {
+  XVerifyViewsFlip flip(&_page_allocator);
+  XAddress::flip_to_remapped();
+}
+
+void XHeap::mark_start() {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+
+  // Verification
+  ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_strong);
+
+  if (XHeap::heap()->has_alloc_stalled()) {
+    // If there are stalled allocations, ensure that regardless of the
+    // cause of the GC, we have to clear soft references, as we are just
+    // about to increment the sequence number, and all previous allocations
+    // will throw if not presented with enough memory.
+    XHeap::heap()->set_soft_reference_policy(true);
+  }
+
+  // Flip address view
+  flip_to_marked();
+
+  // Retire allocating pages
+  _object_allocator.retire_pages();
+
+  // Reset allocated/reclaimed/used statistics
+  _page_allocator.reset_statistics();
+
+  // Reset encountered/dropped/enqueued statistics
+  _reference_processor.reset_statistics();
+
+  // Enter mark phase
+  XGlobalPhase = XPhaseMark;
+
+  // Reset marking information and mark roots
+  _mark.start();
+
+  // Update statistics
+  XStatHeap::set_at_mark_start(_page_allocator.stats());
+}
+
+void XHeap::mark(bool initial) {
+  _mark.mark(initial);
+}
+
+void XHeap::mark_flush_and_free(Thread* thread) {
+  _mark.flush_and_free(thread);
+}
+
+bool XHeap::mark_end() {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+
+  // Try end marking
+  if (!_mark.end()) {
+    // Marking not completed, continue concurrent mark
+    return false;
+  }
+
+  // Enter mark completed phase
+  XGlobalPhase = XPhaseMarkCompleted;
+
+  // Verify after mark
+  XVerify::after_mark();
+
+  // Update statistics
+  XStatHeap::set_at_mark_end(_page_allocator.stats());
+
+  // Block resurrection of weak/phantom references
+  XResurrection::block();
+
+  // Prepare to unload stale metadata and nmethods
+  _unload.prepare();
+
+  // Notify JVMTI that some tagmap entry objects may have died.
+  JvmtiTagMap::set_needs_cleaning();
+
+  return true;
+}
+
+void XHeap::mark_free() {
+  _mark.free();
+}
+
+void XHeap::keep_alive(oop obj) {
+  XBarrier::keep_alive_barrier_on_oop(obj);
+}
+
+void XHeap::set_soft_reference_policy(bool clear) {
+  _reference_processor.set_soft_reference_policy(clear);
+}
+
+class XRendezvousClosure : public HandshakeClosure {
+public:
+  XRendezvousClosure() :
+      HandshakeClosure("XRendezvous") {}
+
+  void do_thread(Thread* thread) {}
+};
+
+void XHeap::process_non_strong_references() {
+  // Process Soft/Weak/Final/PhantomReferences
+  _reference_processor.process_references();
+
+  // Process weak roots
+  _weak_roots_processor.process_weak_roots();
+
+  // Unlink stale metadata and nmethods
+  _unload.unlink();
+
+  // Perform a handshake. This is needed 1) to make sure that stale
+  // metadata and nmethods are no longer observable. And 2), to
+  // prevent the race where a mutator first loads an oop, which is
+  // logically null but not yet cleared. Then this oop gets cleared
+  // by the reference processor and resurrection is unblocked. At
+  // this point the mutator could see the unblocked state and pass
+  // this invalid oop through the normal barrier path, which would
+  // incorrectly try to mark the oop.
+  XRendezvousClosure cl;
+  Handshake::execute(&cl);
+
+  // Unblock resurrection of weak/phantom references
+  XResurrection::unblock();
+
+  // Purge stale metadata and nmethods that were unlinked
+  _unload.purge();
+
+  // Enqueue Soft/Weak/Final/PhantomReferences. Note that this
+  // must be done after unblocking resurrection. Otherwise the
+  // Finalizer thread could call Reference.get() on the Finalizers
+  // that were just enqueued, which would incorrectly return null
+  // during the resurrection block window, since such referents
+  // are only Finalizable marked.
+  _reference_processor.enqueue_references();
+
+  // Clear old markings claim bits.
+  // Note: Clearing _claim_strong also clears _claim_finalizable.
+  ClassLoaderDataGraph::clear_claimed_marks(ClassLoaderData::_claim_strong);
+}
+
+void XHeap::free_empty_pages(XRelocationSetSelector* selector, int bulk) {
+  // Freeing empty pages in bulk is an optimization to avoid grabbing
+  // the page allocator lock, and trying to satisfy stalled allocations
+  // too frequently.
+  if (selector->should_free_empty_pages(bulk)) {
+    free_pages(selector->empty_pages(), true /* reclaimed */);
+    selector->clear_empty_pages();
+  }
+}
+
+void XHeap::select_relocation_set() {
+  // Do not allow pages to be deleted
+  _page_allocator.enable_deferred_delete();
+
+  // Register relocatable pages with selector
+  XRelocationSetSelector selector;
+  XPageTableIterator pt_iter(&_page_table);
+  for (XPage* page; pt_iter.next(&page);) {
+    if (!page->is_relocatable()) {
+      // Not relocatable, don't register
+      continue;
+    }
+
+    if (page->is_marked()) {
+      // Register live page
+      selector.register_live_page(page);
+    } else {
+      // Register empty page
+      selector.register_empty_page(page);
+
+      // Reclaim empty pages in bulk
+      free_empty_pages(&selector, 64 /* bulk */);
+    }
+  }
+
+  // Reclaim remaining empty pages
+  free_empty_pages(&selector, 0 /* bulk */);
+
+  // Allow pages to be deleted
+  _page_allocator.disable_deferred_delete();
+
+  // Select relocation set
+  selector.select();
+
+  // Install relocation set
+  _relocation_set.install(&selector);
+
+  // Setup forwarding table
+  XRelocationSetIterator rs_iter(&_relocation_set);
+  for (XForwarding* forwarding; rs_iter.next(&forwarding);) {
+    _forwarding_table.insert(forwarding);
+  }
+
+  // Update statistics
+  XStatRelocation::set_at_select_relocation_set(selector.stats());
+  XStatHeap::set_at_select_relocation_set(selector.stats());
+}
+
+void XHeap::reset_relocation_set() {
+  // Reset forwarding table
+  XRelocationSetIterator iter(&_relocation_set);
+  for (XForwarding* forwarding; iter.next(&forwarding);) {
+    _forwarding_table.remove(forwarding);
+  }
+
+  // Reset relocation set
+  _relocation_set.reset();
+}
+
+void XHeap::relocate_start() {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+
+  // Finish unloading stale metadata and nmethods
+  _unload.finish();
+
+  // Flip address view
+  flip_to_remapped();
+
+  // Enter relocate phase
+  XGlobalPhase = XPhaseRelocate;
+
+  // Update statistics
+  XStatHeap::set_at_relocate_start(_page_allocator.stats());
+}
+
+void XHeap::relocate() {
+  // Relocate relocation set
+  _relocate.relocate(&_relocation_set);
+
+  // Update statistics
+  XStatHeap::set_at_relocate_end(_page_allocator.stats(), _object_allocator.relocated());
+}
+
+bool XHeap::is_allocating(uintptr_t addr) const {
+  const XPage* const page = _page_table.get(addr);
+  return page->is_allocating();
+}
+
+void XHeap::object_iterate(ObjectClosure* cl, bool visit_weaks) {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+  XHeapIterator iter(1 /* nworkers */, visit_weaks);
+  iter.object_iterate(cl, 0 /* worker_id */);
+}
+
+ParallelObjectIteratorImpl* XHeap::parallel_object_iterator(uint nworkers, bool visit_weaks) {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+  return new XHeapIterator(nworkers, visit_weaks);
+}
+
+void XHeap::pages_do(XPageClosure* cl) {
+  XPageTableIterator iter(&_page_table);
+  for (XPage* page; iter.next(&page);) {
+    cl->do_page(page);
+  }
+  _page_allocator.pages_do(cl);
+}
+
+void XHeap::serviceability_initialize() {
+  _serviceability.initialize();
+}
+
+GCMemoryManager* XHeap::serviceability_cycle_memory_manager() {
+  return _serviceability.cycle_memory_manager();
+}
+
+GCMemoryManager* XHeap::serviceability_pause_memory_manager() {
+  return _serviceability.pause_memory_manager();
+}
+
+MemoryPool* XHeap::serviceability_memory_pool() {
+  return _serviceability.memory_pool();
+}
+
+XServiceabilityCounters* XHeap::serviceability_counters() {
+  return _serviceability.counters();
+}
+
+void XHeap::print_on(outputStream* st) const {
+  st->print_cr(" ZHeap           used " SIZE_FORMAT "M, capacity " SIZE_FORMAT "M, max capacity " SIZE_FORMAT "M",
+               used() / M,
+               capacity() / M,
+               max_capacity() / M);
+  MetaspaceUtils::print_on(st);
+}
+
+void XHeap::print_extended_on(outputStream* st) const {
+  print_on(st);
+  st->cr();
+
+  // Do not allow pages to be deleted
+  _page_allocator.enable_deferred_delete();
+
+  // Print all pages
+  st->print_cr("ZGC Page Table:");
+  XPageTableIterator iter(&_page_table);
+  for (XPage* page; iter.next(&page);) {
+    page->print_on(st);
+  }
+
+  // Allow pages to be deleted
+  _page_allocator.disable_deferred_delete();
+}
+
+bool XHeap::print_location(outputStream* st, uintptr_t addr) const {
+  if (LocationPrinter::is_valid_obj((void*)addr)) {
+    st->print(PTR_FORMAT " is a %s oop: ", addr, XAddress::is_good(addr) ? "good" : "bad");
+    XOop::from_address(addr)->print_on(st);
+    return true;
+  }
+
+  return false;
+}
+
+void XHeap::verify() {
+  // Heap verification can only be done between mark end and
+  // relocate start. This is the only window where all oop are
+  // good and the whole heap is in a consistent state.
+  guarantee(XGlobalPhase == XPhaseMarkCompleted, "Invalid phase");
+
+  XVerify::after_weak_processing();
+}
diff --git a/src/hotspot/share/gc/x/xHeap.hpp b/src/hotspot/share/gc/x/xHeap.hpp
new file mode 100644
index 00000000000..af2c73180d9
--- /dev/null
+++ b/src/hotspot/share/gc/x/xHeap.hpp
@@ -0,0 +1,167 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XHEAP_HPP
+#define SHARE_GC_X_XHEAP_HPP
+
+#include "gc/x/xAllocationFlags.hpp"
+#include "gc/x/xArray.hpp"
+#include "gc/x/xForwardingTable.hpp"
+#include "gc/x/xMark.hpp"
+#include "gc/x/xObjectAllocator.hpp"
+#include "gc/x/xPageAllocator.hpp"
+#include "gc/x/xPageTable.hpp"
+#include "gc/x/xReferenceProcessor.hpp"
+#include "gc/x/xRelocate.hpp"
+#include "gc/x/xRelocationSet.hpp"
+#include "gc/x/xWeakRootsProcessor.hpp"
+#include "gc/x/xServiceability.hpp"
+#include "gc/x/xUnload.hpp"
+#include "gc/x/xWorkers.hpp"
+
+class ThreadClosure;
+class VMStructs;
+class XPage;
+class XRelocationSetSelector;
+
+class XHeap {
+  friend class ::VMStructs;
+
+private:
+  static XHeap*       _heap;
+
+  XWorkers            _workers;
+  XObjectAllocator    _object_allocator;
+  XPageAllocator      _page_allocator;
+  XPageTable          _page_table;
+  XForwardingTable    _forwarding_table;
+  XMark               _mark;
+  XReferenceProcessor _reference_processor;
+  XWeakRootsProcessor _weak_roots_processor;
+  XRelocate           _relocate;
+  XRelocationSet      _relocation_set;
+  XUnload             _unload;
+  XServiceability     _serviceability;
+
+  void flip_to_marked();
+  void flip_to_remapped();
+
+  void free_empty_pages(XRelocationSetSelector* selector, int bulk);
+
+  void out_of_memory();
+
+public:
+  static XHeap* heap();
+
+  XHeap();
+
+  bool is_initialized() const;
+
+  // Heap metrics
+  size_t min_capacity() const;
+  size_t max_capacity() const;
+  size_t soft_max_capacity() const;
+  size_t capacity() const;
+  size_t used() const;
+  size_t unused() const;
+
+  size_t tlab_capacity() const;
+  size_t tlab_used() const;
+  size_t max_tlab_size() const;
+  size_t unsafe_max_tlab_alloc() const;
+
+  bool is_in(uintptr_t addr) const;
+
+  // Threads
+  uint active_workers() const;
+  void set_active_workers(uint nworkers);
+  void threads_do(ThreadClosure* tc) const;
+
+  // Reference processing
+  ReferenceDiscoverer* reference_discoverer();
+  void set_soft_reference_policy(bool clear);
+
+  // Non-strong reference processing
+  void process_non_strong_references();
+
+  // Page allocation
+  XPage* alloc_page(uint8_t type, size_t size, XAllocationFlags flags);
+  void undo_alloc_page(XPage* page);
+  void free_page(XPage* page, bool reclaimed);
+  void free_pages(const XArray<XPage*>* pages, bool reclaimed);
+
+  // Object allocation
+  uintptr_t alloc_tlab(size_t size);
+  uintptr_t alloc_object(size_t size);
+  uintptr_t alloc_object_for_relocation(size_t size);
+  void undo_alloc_object_for_relocation(uintptr_t addr, size_t size);
+  bool has_alloc_stalled() const;
+  void check_out_of_memory();
+
+  // Marking
+  bool is_object_live(uintptr_t addr) const;
+  bool is_object_strongly_live(uintptr_t addr) const;
+  template <bool gc_thread, bool follow, bool finalizable, bool publish> void mark_object(uintptr_t addr);
+  void mark_start();
+  void mark(bool initial);
+  void mark_flush_and_free(Thread* thread);
+  bool mark_end();
+  void mark_free();
+  void keep_alive(oop obj);
+
+  // Relocation set
+  void select_relocation_set();
+  void reset_relocation_set();
+
+  // Relocation
+  void relocate_start();
+  uintptr_t relocate_object(uintptr_t addr);
+  uintptr_t remap_object(uintptr_t addr);
+  void relocate();
+
+  // Continuations
+  bool is_allocating(uintptr_t addr) const;
+
+  // Iteration
+  void object_iterate(ObjectClosure* cl, bool visit_weaks);
+  ParallelObjectIteratorImpl* parallel_object_iterator(uint nworkers, bool visit_weaks);
+  void pages_do(XPageClosure* cl);
+
+  // Serviceability
+  void serviceability_initialize();
+  GCMemoryManager* serviceability_cycle_memory_manager();
+  GCMemoryManager* serviceability_pause_memory_manager();
+  MemoryPool* serviceability_memory_pool();
+  XServiceabilityCounters* serviceability_counters();
+
+  // Printing
+  void print_on(outputStream* st) const;
+  void print_extended_on(outputStream* st) const;
+  bool print_location(outputStream* st, uintptr_t addr) const;
+
+  // Verification
+  bool is_oop(uintptr_t addr) const;
+  void verify();
+};
+
+#endif // SHARE_GC_X_XHEAP_HPP
diff --git a/src/hotspot/share/gc/x/xHeap.inline.hpp b/src/hotspot/share/gc/x/xHeap.inline.hpp
new file mode 100644
index 00000000000..5b3e06b2f4b
--- /dev/null
+++ b/src/hotspot/share/gc/x/xHeap.inline.hpp
@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XHEAP_INLINE_HPP
+#define SHARE_GC_X_XHEAP_INLINE_HPP
+
+#include "gc/x/xHeap.hpp"
+
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xForwardingTable.inline.hpp"
+#include "gc/x/xMark.inline.hpp"
+#include "gc/x/xPage.inline.hpp"
+#include "gc/x/xPageTable.inline.hpp"
+#include "utilities/debug.hpp"
+
+inline XHeap* XHeap::heap() {
+  assert(_heap != NULL, "Not initialized");
+  return _heap;
+}
+
+inline ReferenceDiscoverer* XHeap::reference_discoverer() {
+  return &_reference_processor;
+}
+
+inline bool XHeap::is_object_live(uintptr_t addr) const {
+  XPage* page = _page_table.get(addr);
+  return page->is_object_live(addr);
+}
+
+inline bool XHeap::is_object_strongly_live(uintptr_t addr) const {
+  XPage* page = _page_table.get(addr);
+  return page->is_object_strongly_live(addr);
+}
+
+template <bool gc_thread, bool follow, bool finalizable, bool publish>
+inline void XHeap::mark_object(uintptr_t addr) {
+  assert(XGlobalPhase == XPhaseMark, "Mark not allowed");
+  _mark.mark_object<gc_thread, follow, finalizable, publish>(addr);
+}
+
+inline uintptr_t XHeap::alloc_tlab(size_t size) {
+  guarantee(size <= max_tlab_size(), "TLAB too large");
+  return _object_allocator.alloc_object(size);
+}
+
+inline uintptr_t XHeap::alloc_object(size_t size) {
+  uintptr_t addr = _object_allocator.alloc_object(size);
+  assert(XAddress::is_good_or_null(addr), "Bad address");
+
+  if (addr == 0) {
+    out_of_memory();
+  }
+
+  return addr;
+}
+
+inline uintptr_t XHeap::alloc_object_for_relocation(size_t size) {
+  const uintptr_t addr = _object_allocator.alloc_object_for_relocation(&_page_table, size);
+  assert(XAddress::is_good_or_null(addr), "Bad address");
+  return addr;
+}
+
+inline void XHeap::undo_alloc_object_for_relocation(uintptr_t addr, size_t size) {
+  XPage* const page = _page_table.get(addr);
+  _object_allocator.undo_alloc_object_for_relocation(page, addr, size);
+}
+
+inline uintptr_t XHeap::relocate_object(uintptr_t addr) {
+  assert(XGlobalPhase == XPhaseRelocate, "Relocate not allowed");
+
+  XForwarding* const forwarding = _forwarding_table.get(addr);
+  if (forwarding == NULL) {
+    // Not forwarding
+    return XAddress::good(addr);
+  }
+
+  // Relocate object
+  return _relocate.relocate_object(forwarding, XAddress::good(addr));
+}
+
+inline uintptr_t XHeap::remap_object(uintptr_t addr) {
+  assert(XGlobalPhase == XPhaseMark ||
+         XGlobalPhase == XPhaseMarkCompleted, "Forward not allowed");
+
+  XForwarding* const forwarding = _forwarding_table.get(addr);
+  if (forwarding == NULL) {
+    // Not forwarding
+    return XAddress::good(addr);
+  }
+
+  // Forward object
+  return _relocate.forward_object(forwarding, XAddress::good(addr));
+}
+
+inline bool XHeap::has_alloc_stalled() const {
+  return _page_allocator.has_alloc_stalled();
+}
+
+inline void XHeap::check_out_of_memory() {
+  _page_allocator.check_out_of_memory();
+}
+
+inline bool XHeap::is_oop(uintptr_t addr) const {
+  return XAddress::is_good(addr) && is_object_aligned(addr) && is_in(addr);
+}
+
+#endif // SHARE_GC_X_XHEAP_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xHeapIterator.cpp b/src/hotspot/share/gc/x/xHeapIterator.cpp
new file mode 100644
index 00000000000..614f0089356
--- /dev/null
+++ b/src/hotspot/share/gc/x/xHeapIterator.cpp
@@ -0,0 +1,439 @@
+/*
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "classfile/classLoaderDataGraph.hpp"
+#include "gc/shared/barrierSetNMethod.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/shared/taskqueue.inline.hpp"
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xCollectedHeap.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xGranuleMap.inline.hpp"
+#include "gc/x/xHeapIterator.hpp"
+#include "gc/x/xLock.inline.hpp"
+#include "gc/x/xNMethod.hpp"
+#include "gc/x/xOop.inline.hpp"
+#include "memory/iterator.inline.hpp"
+#include "utilities/bitMap.inline.hpp"
+
+class XHeapIteratorBitMap : public CHeapObj<mtGC> {
+private:
+  CHeapBitMap _bitmap;
+
+public:
+  XHeapIteratorBitMap(size_t size_in_bits) :
+      _bitmap(size_in_bits, mtGC) {}
+
+  bool try_set_bit(size_t index) {
+    return _bitmap.par_set_bit(index);
+  }
+};
+
+class XHeapIteratorContext {
+private:
+  XHeapIterator* const           _iter;
+  XHeapIteratorQueue* const      _queue;
+  XHeapIteratorArrayQueue* const _array_queue;
+  const uint                     _worker_id;
+  XStatTimerDisable              _timer_disable;
+
+public:
+  XHeapIteratorContext(XHeapIterator* iter, uint worker_id) :
+      _iter(iter),
+      _queue(_iter->_queues.queue(worker_id)),
+      _array_queue(_iter->_array_queues.queue(worker_id)),
+      _worker_id(worker_id) {}
+
+  void mark_and_push(oop obj) const {
+    if (_iter->mark_object(obj)) {
+      _queue->push(obj);
+    }
+  }
+
+  void push_array(const ObjArrayTask& array) const {
+    _array_queue->push(array);
+  }
+
+  bool pop(oop& obj) const {
+    return _queue->pop_overflow(obj) || _queue->pop_local(obj);
+  }
+
+  bool pop_array(ObjArrayTask& array) const {
+    return _array_queue->pop_overflow(array) || _array_queue->pop_local(array);
+  }
+
+  bool steal(oop& obj) const {
+    return _iter->_queues.steal(_worker_id, obj);
+  }
+
+  bool steal_array(ObjArrayTask& array) const {
+    return _iter->_array_queues.steal(_worker_id, array);
+  }
+
+  bool is_drained() const {
+    return _queue->is_empty() && _array_queue->is_empty();
+  }
+};
+
+template <bool Weak>
+class XHeapIteratorRootOopClosure : public OopClosure {
+private:
+  const XHeapIteratorContext& _context;
+
+  oop load_oop(oop* p) {
+    if (Weak) {
+      return NativeAccess<AS_NO_KEEPALIVE | ON_PHANTOM_OOP_REF>::oop_load(p);
+    }
+
+    return NativeAccess<AS_NO_KEEPALIVE>::oop_load(p);
+  }
+
+public:
+  XHeapIteratorRootOopClosure(const XHeapIteratorContext& context) :
+      _context(context) {}
+
+  virtual void do_oop(oop* p) {
+    const oop obj = load_oop(p);
+    _context.mark_and_push(obj);
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+};
+
+template <bool VisitReferents>
+class XHeapIteratorOopClosure : public OopIterateClosure {
+private:
+  const XHeapIteratorContext& _context;
+  const oop                   _base;
+
+  oop load_oop(oop* p) {
+    assert(XCollectedHeap::heap()->is_in(p), "Should be in heap");
+
+    if (VisitReferents) {
+      return HeapAccess<AS_NO_KEEPALIVE | ON_UNKNOWN_OOP_REF>::oop_load_at(_base, _base->field_offset(p));
+    }
+
+    return HeapAccess<AS_NO_KEEPALIVE>::oop_load(p);
+  }
+
+public:
+  XHeapIteratorOopClosure(const XHeapIteratorContext& context, oop base) :
+      OopIterateClosure(),
+      _context(context),
+      _base(base) {}
+
+  virtual ReferenceIterationMode reference_iteration_mode() {
+    return VisitReferents ? DO_FIELDS : DO_FIELDS_EXCEPT_REFERENT;
+  }
+
+  virtual void do_oop(oop* p) {
+    const oop obj = load_oop(p);
+    _context.mark_and_push(obj);
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+
+  virtual bool do_metadata() {
+    return true;
+  }
+
+  virtual void do_klass(Klass* k) {
+    ClassLoaderData* const cld = k->class_loader_data();
+    XHeapIteratorOopClosure::do_cld(cld);
+  }
+
+  virtual void do_cld(ClassLoaderData* cld) {
+    class NativeAccessClosure : public OopClosure {
+    private:
+      const XHeapIteratorContext& _context;
+
+    public:
+      explicit NativeAccessClosure(const XHeapIteratorContext& context) :
+          _context(context) {}
+
+      virtual void do_oop(oop* p) {
+        assert(!XCollectedHeap::heap()->is_in(p), "Should not be in heap");
+        const oop obj = NativeAccess<AS_NO_KEEPALIVE>::oop_load(p);
+        _context.mark_and_push(obj);
+      }
+
+      virtual void do_oop(narrowOop* p) {
+        ShouldNotReachHere();
+      }
+    };
+
+    NativeAccessClosure cl(_context);
+    cld->oops_do(&cl, ClassLoaderData::_claim_other);
+  }
+
+  // Don't follow loom stack metadata; it's already followed in other ways through CLDs
+  virtual void do_nmethod(nmethod* nm) {}
+  virtual void do_method(Method* m) {}
+};
+
+XHeapIterator::XHeapIterator(uint nworkers, bool visit_weaks) :
+    _visit_weaks(visit_weaks),
+    _timer_disable(),
+    _bitmaps(XAddressOffsetMax),
+    _bitmaps_lock(),
+    _queues(nworkers),
+    _array_queues(nworkers),
+    _roots(ClassLoaderData::_claim_other),
+    _weak_roots(),
+    _terminator(nworkers, &_queues) {
+
+  // Create queues
+  for (uint i = 0; i < _queues.size(); i++) {
+    XHeapIteratorQueue* const queue = new XHeapIteratorQueue();
+    _queues.register_queue(i, queue);
+  }
+
+  // Create array queues
+  for (uint i = 0; i < _array_queues.size(); i++) {
+    XHeapIteratorArrayQueue* const array_queue = new XHeapIteratorArrayQueue();
+    _array_queues.register_queue(i, array_queue);
+  }
+}
+
+XHeapIterator::~XHeapIterator() {
+  // Destroy bitmaps
+  XHeapIteratorBitMapsIterator iter(&_bitmaps);
+  for (XHeapIteratorBitMap* bitmap; iter.next(&bitmap);) {
+    delete bitmap;
+  }
+
+  // Destroy array queues
+  for (uint i = 0; i < _array_queues.size(); i++) {
+    delete _array_queues.queue(i);
+  }
+
+  // Destroy queues
+  for (uint i = 0; i < _queues.size(); i++) {
+    delete _queues.queue(i);
+  }
+
+  // Clear claimed CLD bits
+  ClassLoaderDataGraph::clear_claimed_marks(ClassLoaderData::_claim_other);
+}
+
+static size_t object_index_max() {
+  return XGranuleSize >> XObjectAlignmentSmallShift;
+}
+
+static size_t object_index(oop obj) {
+  const uintptr_t addr = XOop::to_address(obj);
+  const uintptr_t offset = XAddress::offset(addr);
+  const uintptr_t mask = XGranuleSize - 1;
+  return (offset & mask) >> XObjectAlignmentSmallShift;
+}
+
+XHeapIteratorBitMap* XHeapIterator::object_bitmap(oop obj) {
+  const uintptr_t offset = XAddress::offset(XOop::to_address(obj));
+  XHeapIteratorBitMap* bitmap = _bitmaps.get_acquire(offset);
+  if (bitmap == NULL) {
+    XLocker<XLock> locker(&_bitmaps_lock);
+    bitmap = _bitmaps.get(offset);
+    if (bitmap == NULL) {
+      // Install new bitmap
+      bitmap = new XHeapIteratorBitMap(object_index_max());
+      _bitmaps.release_put(offset, bitmap);
+    }
+  }
+
+  return bitmap;
+}
+
+bool XHeapIterator::mark_object(oop obj) {
+  if (obj == NULL) {
+    return false;
+  }
+
+  XHeapIteratorBitMap* const bitmap = object_bitmap(obj);
+  const size_t index = object_index(obj);
+  return bitmap->try_set_bit(index);
+}
+
+typedef ClaimingCLDToOopClosure<ClassLoaderData::_claim_other> XHeapIteratorCLDCLosure;
+
+class XHeapIteratorNMethodClosure : public NMethodClosure {
+private:
+  OopClosure* const        _cl;
+  BarrierSetNMethod* const _bs_nm;
+
+public:
+  XHeapIteratorNMethodClosure(OopClosure* cl) :
+      _cl(cl),
+      _bs_nm(BarrierSet::barrier_set()->barrier_set_nmethod()) {}
+
+  virtual void do_nmethod(nmethod* nm) {
+    // If ClassUnloading is turned off, all nmethods are considered strong,
+    // not only those on the call stacks. The heap iteration might happen
+    // before the concurrent processign of the code cache, make sure that
+    // all nmethods have been processed before visiting the oops.
+    _bs_nm->nmethod_entry_barrier(nm);
+
+    XNMethod::nmethod_oops_do(nm, _cl);
+  }
+};
+
+class XHeapIteratorThreadClosure : public ThreadClosure {
+private:
+  OopClosure* const        _cl;
+  CodeBlobToNMethodClosure _cb_cl;
+
+public:
+  XHeapIteratorThreadClosure(OopClosure* cl, NMethodClosure* nm_cl) :
+      _cl(cl),
+      _cb_cl(nm_cl) {}
+
+  void do_thread(Thread* thread) {
+    thread->oops_do(_cl, &_cb_cl);
+  }
+};
+
+void XHeapIterator::push_strong_roots(const XHeapIteratorContext& context) {
+  XHeapIteratorRootOopClosure<false /* Weak */> cl(context);
+  XHeapIteratorCLDCLosure cld_cl(&cl);
+  XHeapIteratorNMethodClosure nm_cl(&cl);
+  XHeapIteratorThreadClosure thread_cl(&cl, &nm_cl);
+
+  _roots.apply(&cl,
+               &cld_cl,
+               &thread_cl,
+               &nm_cl);
+}
+
+void XHeapIterator::push_weak_roots(const XHeapIteratorContext& context) {
+  XHeapIteratorRootOopClosure<true  /* Weak */> cl(context);
+  _weak_roots.apply(&cl);
+}
+
+template <bool VisitWeaks>
+void XHeapIterator::push_roots(const XHeapIteratorContext& context) {
+  push_strong_roots(context);
+  if (VisitWeaks) {
+    push_weak_roots(context);
+  }
+}
+
+template <bool VisitReferents>
+void XHeapIterator::follow_object(const XHeapIteratorContext& context, oop obj) {
+  XHeapIteratorOopClosure<VisitReferents> cl(context, obj);
+  obj->oop_iterate(&cl);
+}
+
+void XHeapIterator::follow_array(const XHeapIteratorContext& context, oop obj) {
+  // Follow klass
+  XHeapIteratorOopClosure<false /* VisitReferents */> cl(context, obj);
+  cl.do_klass(obj->klass());
+
+  // Push array chunk
+  context.push_array(ObjArrayTask(obj, 0 /* index */));
+}
+
+void XHeapIterator::follow_array_chunk(const XHeapIteratorContext& context, const ObjArrayTask& array) {
+  const objArrayOop obj = objArrayOop(array.obj());
+  const int length = obj->length();
+  const int start = array.index();
+  const int stride = MIN2<int>(length - start, ObjArrayMarkingStride);
+  const int end = start + stride;
+
+  // Push remaining array chunk first
+  if (end < length) {
+    context.push_array(ObjArrayTask(obj, end));
+  }
+
+  // Follow array chunk
+  XHeapIteratorOopClosure<false /* VisitReferents */> cl(context, obj);
+  obj->oop_iterate_range(&cl, start, end);
+}
+
+template <bool VisitWeaks>
+void XHeapIterator::visit_and_follow(const XHeapIteratorContext& context, ObjectClosure* cl, oop obj) {
+  // Visit
+  cl->do_object(obj);
+
+  // Follow
+  if (obj->is_objArray()) {
+    follow_array(context, obj);
+  } else {
+    follow_object<VisitWeaks>(context, obj);
+  }
+}
+
+template <bool VisitWeaks>
+void XHeapIterator::drain(const XHeapIteratorContext& context, ObjectClosure* cl) {
+  ObjArrayTask array;
+  oop obj;
+
+  do {
+    while (context.pop(obj)) {
+      visit_and_follow<VisitWeaks>(context, cl, obj);
+    }
+
+    if (context.pop_array(array)) {
+      follow_array_chunk(context, array);
+    }
+  } while (!context.is_drained());
+}
+
+template <bool VisitWeaks>
+void XHeapIterator::steal(const XHeapIteratorContext& context, ObjectClosure* cl) {
+  ObjArrayTask array;
+  oop obj;
+
+  if (context.steal_array(array)) {
+    follow_array_chunk(context, array);
+  } else if (context.steal(obj)) {
+    visit_and_follow<VisitWeaks>(context, cl, obj);
+  }
+}
+
+template <bool VisitWeaks>
+void XHeapIterator::drain_and_steal(const XHeapIteratorContext& context, ObjectClosure* cl) {
+  do {
+    drain<VisitWeaks>(context, cl);
+    steal<VisitWeaks>(context, cl);
+  } while (!context.is_drained() || !_terminator.offer_termination());
+}
+
+template <bool VisitWeaks>
+void XHeapIterator::object_iterate_inner(const XHeapIteratorContext& context, ObjectClosure* object_cl) {
+  push_roots<VisitWeaks>(context);
+  drain_and_steal<VisitWeaks>(context, object_cl);
+}
+
+void XHeapIterator::object_iterate(ObjectClosure* cl, uint worker_id) {
+  XHeapIteratorContext context(this, worker_id);
+
+  if (_visit_weaks) {
+    object_iterate_inner<true /* VisitWeaks */>(context, cl);
+  } else {
+    object_iterate_inner<false /* VisitWeaks */>(context, cl);
+  }
+}
diff --git a/src/hotspot/share/gc/x/xHeapIterator.hpp b/src/hotspot/share/gc/x/xHeapIterator.hpp
new file mode 100644
index 00000000000..0d990a616f8
--- /dev/null
+++ b/src/hotspot/share/gc/x/xHeapIterator.hpp
@@ -0,0 +1,97 @@
+/*
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XHEAPITERATOR_HPP
+#define SHARE_GC_X_XHEAPITERATOR_HPP
+
+#include "gc/shared/collectedHeap.hpp"
+#include "gc/shared/taskTerminator.hpp"
+#include "gc/shared/taskqueue.hpp"
+#include "gc/x/xGranuleMap.hpp"
+#include "gc/x/xLock.hpp"
+#include "gc/x/xRootsIterator.hpp"
+#include "gc/x/xStat.hpp"
+
+class XHeapIteratorBitMap;
+class XHeapIteratorContext;
+
+using XHeapIteratorBitMaps = XGranuleMap<XHeapIteratorBitMap*>;
+using XHeapIteratorBitMapsIterator = XGranuleMapIterator<XHeapIteratorBitMap*>;
+using XHeapIteratorQueue = OverflowTaskQueue<oop, mtGC>;
+using XHeapIteratorQueues = GenericTaskQueueSet<XHeapIteratorQueue, mtGC>;
+using XHeapIteratorArrayQueue = OverflowTaskQueue<ObjArrayTask, mtGC>;
+using XHeapIteratorArrayQueues = GenericTaskQueueSet<XHeapIteratorArrayQueue, mtGC>;
+
+class XHeapIterator : public ParallelObjectIteratorImpl {
+  friend class XHeapIteratorContext;
+
+private:
+  const bool               _visit_weaks;
+  XStatTimerDisable        _timer_disable;
+  XHeapIteratorBitMaps     _bitmaps;
+  XLock                    _bitmaps_lock;
+  XHeapIteratorQueues      _queues;
+  XHeapIteratorArrayQueues _array_queues;
+  XRootsIterator           _roots;
+  XWeakRootsIterator       _weak_roots;
+  TaskTerminator           _terminator;
+
+  XHeapIteratorBitMap* object_bitmap(oop obj);
+
+  bool mark_object(oop obj);
+
+  void push_strong_roots(const XHeapIteratorContext& context);
+  void push_weak_roots(const XHeapIteratorContext& context);
+
+  template <bool VisitWeaks>
+  void push_roots(const XHeapIteratorContext& context);
+
+  template <bool VisitReferents>
+  void follow_object(const XHeapIteratorContext& context, oop obj);
+
+  void follow_array(const XHeapIteratorContext& context, oop obj);
+  void follow_array_chunk(const XHeapIteratorContext& context, const ObjArrayTask& array);
+
+  template <bool VisitWeaks>
+  void visit_and_follow(const XHeapIteratorContext& context, ObjectClosure* cl, oop obj);
+
+  template <bool VisitWeaks>
+  void drain(const XHeapIteratorContext& context, ObjectClosure* cl);
+
+  template <bool VisitWeaks>
+  void steal(const XHeapIteratorContext& context, ObjectClosure* cl);
+
+  template <bool VisitWeaks>
+  void drain_and_steal(const XHeapIteratorContext& context, ObjectClosure* cl);
+
+  template <bool VisitWeaks>
+  void object_iterate_inner(const XHeapIteratorContext& context, ObjectClosure* cl);
+
+public:
+  XHeapIterator(uint nworkers, bool visit_weaks);
+  virtual ~XHeapIterator();
+
+  virtual void object_iterate(ObjectClosure* cl, uint worker_id);
+};
+
+#endif // SHARE_GC_X_XHEAPITERATOR_HPP
diff --git a/src/hotspot/share/gc/x/xHeuristics.cpp b/src/hotspot/share/gc/x/xHeuristics.cpp
new file mode 100644
index 00000000000..ec89fa41919
--- /dev/null
+++ b/src/hotspot/share/gc/x/xHeuristics.cpp
@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/xCPU.inline.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xHeuristics.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/os.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+void XHeuristics::set_medium_page_size() {
+  // Set XPageSizeMedium so that a medium page occupies at most 3.125% of the
+  // max heap size. XPageSizeMedium is initially set to 0, which means medium
+  // pages are effectively disabled. It is adjusted only if XPageSizeMedium
+  // becomes larger than XPageSizeSmall.
+  const size_t min = XGranuleSize;
+  const size_t max = XGranuleSize * 16;
+  const size_t unclamped = MaxHeapSize * 0.03125;
+  const size_t clamped = clamp(unclamped, min, max);
+  const size_t size = round_down_power_of_2(clamped);
+
+  if (size > XPageSizeSmall) {
+    // Enable medium pages
+    XPageSizeMedium             = size;
+    XPageSizeMediumShift        = log2i_exact(XPageSizeMedium);
+    XObjectSizeLimitMedium      = XPageSizeMedium / 8;
+    XObjectAlignmentMediumShift = (int)XPageSizeMediumShift - 13;
+    XObjectAlignmentMedium      = 1 << XObjectAlignmentMediumShift;
+  }
+}
+
+size_t XHeuristics::relocation_headroom() {
+  // Calculate headroom needed to avoid in-place relocation. Each worker will try
+  // to allocate a small page, and all workers will share a single medium page.
+  const uint nworkers = UseDynamicNumberOfGCThreads ? ConcGCThreads : MAX2(ConcGCThreads, ParallelGCThreads);
+  return (nworkers * XPageSizeSmall) + XPageSizeMedium;
+}
+
+bool XHeuristics::use_per_cpu_shared_small_pages() {
+  // Use per-CPU shared small pages only if these pages occupy at most 3.125%
+  // of the max heap size. Otherwise fall back to using a single shared small
+  // page. This is useful when using small heaps on large machines.
+  const size_t per_cpu_share = (MaxHeapSize * 0.03125) / XCPU::count();
+  return per_cpu_share >= XPageSizeSmall;
+}
+
+static uint nworkers_based_on_ncpus(double cpu_share_in_percent) {
+  return ceil(os::initial_active_processor_count() * cpu_share_in_percent / 100.0);
+}
+
+static uint nworkers_based_on_heap_size(double heap_share_in_percent) {
+  const int nworkers = (MaxHeapSize * (heap_share_in_percent / 100.0)) / XPageSizeSmall;
+  return MAX2(nworkers, 1);
+}
+
+static uint nworkers(double cpu_share_in_percent) {
+  // Cap number of workers so that they don't use more than 2% of the max heap
+  // during relocation. This is useful when using small heaps on large machines.
+  return MIN2(nworkers_based_on_ncpus(cpu_share_in_percent),
+              nworkers_based_on_heap_size(2.0));
+}
+
+uint XHeuristics::nparallel_workers() {
+  // Use 60% of the CPUs, rounded up. We would like to use as many threads as
+  // possible to increase parallelism. However, using a thread count that is
+  // close to the number of processors tends to lead to over-provisioning and
+  // scheduling latency issues. Using 60% of the active processors appears to
+  // be a fairly good balance.
+  return nworkers(60.0);
+}
+
+uint XHeuristics::nconcurrent_workers() {
+  // The number of concurrent threads we would like to use heavily depends
+  // on the type of workload we are running. Using too many threads will have
+  // a negative impact on the application throughput, while using too few
+  // threads will prolong the GC-cycle and we then risk being out-run by the
+  // application. When in dynamic mode, use up to 25% of the active processors.
+  //  When in non-dynamic mode, use 12.5% of the active processors.
+  return nworkers(UseDynamicNumberOfGCThreads ? 25.0 : 12.5);
+}
diff --git a/src/hotspot/share/gc/x/xHeuristics.hpp b/src/hotspot/share/gc/x/xHeuristics.hpp
new file mode 100644
index 00000000000..2ca798257b2
--- /dev/null
+++ b/src/hotspot/share/gc/x/xHeuristics.hpp
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XHEURISTICS_HPP
+#define SHARE_GC_X_XHEURISTICS_HPP
+
+#include "memory/allStatic.hpp"
+
+class XHeuristics : public AllStatic {
+public:
+  static void set_medium_page_size();
+
+  static size_t relocation_headroom();
+
+  static bool use_per_cpu_shared_small_pages();
+
+  static uint nparallel_workers();
+  static uint nconcurrent_workers();
+};
+
+#endif // SHARE_GC_X_XHEURISTICS_HPP
diff --git a/src/hotspot/share/gc/x/xInitialize.cpp b/src/hotspot/share/gc/x/xInitialize.cpp
new file mode 100644
index 00000000000..01b79f3ffd7
--- /dev/null
+++ b/src/hotspot/share/gc/x/xInitialize.cpp
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xAddress.hpp"
+#include "gc/x/xBarrierSet.hpp"
+#include "gc/x/xCPU.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xHeuristics.hpp"
+#include "gc/x/xInitialize.hpp"
+#include "gc/x/xLargePages.hpp"
+#include "gc/x/xNUMA.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xThreadLocalAllocBuffer.hpp"
+#include "gc/x/xTracer.hpp"
+#include "logging/log.hpp"
+#include "runtime/vm_version.hpp"
+
+XInitialize::XInitialize(XBarrierSet* barrier_set) {
+  log_info(gc, init)("Initializing %s", XName);
+  log_info(gc, init)("Version: %s (%s)",
+                     VM_Version::vm_release(),
+                     VM_Version::jdk_debug_level());
+  log_info(gc, init)("Using legacy single-generation mode");
+
+  // Early initialization
+  XAddress::initialize();
+  XNUMA::initialize();
+  XCPU::initialize();
+  XStatValue::initialize();
+  XThreadLocalAllocBuffer::initialize();
+  XTracer::initialize();
+  XLargePages::initialize();
+  XHeuristics::set_medium_page_size();
+  XBarrierSet::set_barrier_set(barrier_set);
+
+  pd_initialize();
+}
diff --git a/src/hotspot/share/gc/x/xInitialize.hpp b/src/hotspot/share/gc/x/xInitialize.hpp
new file mode 100644
index 00000000000..30e7b65293e
--- /dev/null
+++ b/src/hotspot/share/gc/x/xInitialize.hpp
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XINITIALIZE_HPP
+#define SHARE_GC_X_XINITIALIZE_HPP
+
+#include "memory/allocation.hpp"
+
+class XBarrierSet;
+
+class XInitialize {
+private:
+  void pd_initialize();
+
+public:
+  XInitialize(XBarrierSet* barrier_set);
+};
+
+#endif // SHARE_GC_X_XINITIALIZE_HPP
diff --git a/src/hotspot/share/gc/x/xLargePages.cpp b/src/hotspot/share/gc/x/xLargePages.cpp
new file mode 100644
index 00000000000..13da763c6a3
--- /dev/null
+++ b/src/hotspot/share/gc/x/xLargePages.cpp
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/x/xLargePages.hpp"
+#include "runtime/os.hpp"
+
+XLargePages::State XLargePages::_state;
+
+void XLargePages::initialize() {
+  pd_initialize();
+
+  log_info_p(gc, init)("Memory: " JULONG_FORMAT "M", os::physical_memory() / M);
+  log_info_p(gc, init)("Large Page Support: %s", to_string());
+}
+
+const char* XLargePages::to_string() {
+  switch (_state) {
+  case Explicit:
+    return "Enabled (Explicit)";
+
+  case Transparent:
+    return "Enabled (Transparent)";
+
+  default:
+    return "Disabled";
+  }
+}
diff --git a/src/hotspot/share/gc/x/xLargePages.hpp b/src/hotspot/share/gc/x/xLargePages.hpp
new file mode 100644
index 00000000000..562e83ffbd0
--- /dev/null
+++ b/src/hotspot/share/gc/x/xLargePages.hpp
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XLARGEPAGES_HPP
+#define SHARE_GC_X_XLARGEPAGES_HPP
+
+#include "memory/allStatic.hpp"
+
+class XLargePages : public AllStatic {
+private:
+  enum State {
+    Disabled,
+    Explicit,
+    Transparent
+  };
+
+  static State _state;
+
+  static void pd_initialize();
+
+public:
+  static void initialize();
+
+  static bool is_enabled();
+  static bool is_explicit();
+  static bool is_transparent();
+
+  static const char* to_string();
+};
+
+#endif // SHARE_GC_X_XLARGEPAGES_HPP
diff --git a/src/hotspot/share/gc/x/xLargePages.inline.hpp b/src/hotspot/share/gc/x/xLargePages.inline.hpp
new file mode 100644
index 00000000000..2f027c3b176
--- /dev/null
+++ b/src/hotspot/share/gc/x/xLargePages.inline.hpp
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XLARGEPAGES_INLINE_HPP
+#define SHARE_GC_X_XLARGEPAGES_INLINE_HPP
+
+#include "gc/x/xLargePages.hpp"
+
+inline bool XLargePages::is_enabled() {
+  return _state != Disabled;
+}
+
+inline bool XLargePages::is_explicit() {
+  return _state == Explicit;
+}
+
+inline bool XLargePages::is_transparent() {
+  return _state == Transparent;
+}
+
+#endif // SHARE_GC_X_XLARGEPAGES_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xList.hpp b/src/hotspot/share/gc/x/xList.hpp
new file mode 100644
index 00000000000..d689704d653
--- /dev/null
+++ b/src/hotspot/share/gc/x/xList.hpp
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XLIST_HPP
+#define SHARE_GC_X_XLIST_HPP
+
+#include "memory/allocation.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+template <typename T> class XList;
+
+// Element in a doubly linked list
+template <typename T>
+class XListNode {
+  friend class XList<T>;
+
+private:
+  XListNode<T>* _next;
+  XListNode<T>* _prev;
+
+  NONCOPYABLE(XListNode);
+
+  void verify_links() const;
+  void verify_links_linked() const;
+  void verify_links_unlinked() const;
+
+public:
+  XListNode();
+  ~XListNode();
+};
+
+// Doubly linked list
+template <typename T>
+class XList {
+private:
+  XListNode<T> _head;
+  size_t       _size;
+
+  NONCOPYABLE(XList);
+
+  void verify_head() const;
+
+  void insert(XListNode<T>* before, XListNode<T>* node);
+
+  XListNode<T>* cast_to_inner(T* elem) const;
+  T* cast_to_outer(XListNode<T>* node) const;
+
+public:
+  XList();
+
+  size_t size() const;
+  bool is_empty() const;
+
+  T* first() const;
+  T* last() const;
+  T* next(T* elem) const;
+  T* prev(T* elem) const;
+
+  void insert_first(T* elem);
+  void insert_last(T* elem);
+  void insert_before(T* before, T* elem);
+  void insert_after(T* after, T* elem);
+
+  void remove(T* elem);
+  T* remove_first();
+  T* remove_last();
+};
+
+template <typename T, bool Forward>
+class XListIteratorImpl : public StackObj {
+private:
+  const XList<T>* const _list;
+  T*                    _next;
+
+public:
+  XListIteratorImpl(const XList<T>* list);
+
+  bool next(T** elem);
+};
+
+template <typename T, bool Forward>
+class XListRemoveIteratorImpl : public StackObj {
+private:
+  XList<T>* const _list;
+
+public:
+  XListRemoveIteratorImpl(XList<T>* list);
+
+  bool next(T** elem);
+};
+
+template <typename T> using XListIterator = XListIteratorImpl<T, true /* Forward */>;
+template <typename T> using XListReverseIterator = XListIteratorImpl<T, false /* Forward */>;
+template <typename T> using XListRemoveIterator = XListRemoveIteratorImpl<T, true /* Forward */>;
+
+#endif // SHARE_GC_X_XLIST_HPP
diff --git a/src/hotspot/share/gc/x/xList.inline.hpp b/src/hotspot/share/gc/x/xList.inline.hpp
new file mode 100644
index 00000000000..25c28fbda43
--- /dev/null
+++ b/src/hotspot/share/gc/x/xList.inline.hpp
@@ -0,0 +1,238 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XLIST_INLINE_HPP
+#define SHARE_GC_X_XLIST_INLINE_HPP
+
+#include "gc/x/xList.hpp"
+
+#include "utilities/debug.hpp"
+
+template <typename T>
+inline XListNode<T>::XListNode() :
+    _next(this),
+    _prev(this) {}
+
+template <typename T>
+inline XListNode<T>::~XListNode() {
+  verify_links_unlinked();
+}
+
+template <typename T>
+inline void XListNode<T>::verify_links() const {
+  assert(_next->_prev == this, "Corrupt list node");
+  assert(_prev->_next == this, "Corrupt list node");
+}
+
+template <typename T>
+inline void XListNode<T>::verify_links_linked() const {
+  assert(_next != this, "Should be in a list");
+  assert(_prev != this, "Should be in a list");
+  verify_links();
+}
+
+template <typename T>
+inline void XListNode<T>::verify_links_unlinked() const {
+  assert(_next == this, "Should not be in a list");
+  assert(_prev == this, "Should not be in a list");
+}
+
+template <typename T>
+inline void XList<T>::verify_head() const {
+  _head.verify_links();
+}
+
+template <typename T>
+inline void XList<T>::insert(XListNode<T>* before, XListNode<T>* node) {
+  verify_head();
+
+  before->verify_links();
+  node->verify_links_unlinked();
+
+  node->_prev = before;
+  node->_next = before->_next;
+  before->_next = node;
+  node->_next->_prev = node;
+
+  before->verify_links_linked();
+  node->verify_links_linked();
+
+  _size++;
+}
+
+template <typename T>
+inline XListNode<T>* XList<T>::cast_to_inner(T* elem) const {
+  return &elem->_node;
+}
+
+template <typename T>
+inline T* XList<T>::cast_to_outer(XListNode<T>* node) const {
+  return (T*)((uintptr_t)node - offset_of(T, _node));
+}
+
+template <typename T>
+inline XList<T>::XList() :
+    _head(),
+    _size(0) {
+  verify_head();
+}
+
+template <typename T>
+inline size_t XList<T>::size() const {
+  verify_head();
+  return _size;
+}
+
+template <typename T>
+inline bool XList<T>::is_empty() const {
+  return size() == 0;
+}
+
+template <typename T>
+inline T* XList<T>::first() const {
+  return is_empty() ? NULL : cast_to_outer(_head._next);
+}
+
+template <typename T>
+inline T* XList<T>::last() const {
+  return is_empty() ? NULL : cast_to_outer(_head._prev);
+}
+
+template <typename T>
+inline T* XList<T>::next(T* elem) const {
+  verify_head();
+
+  XListNode<T>* const node = cast_to_inner(elem);
+  node->verify_links_linked();
+
+  XListNode<T>* const next = node->_next;
+  next->verify_links_linked();
+
+  return (next == &_head) ? NULL : cast_to_outer(next);
+}
+
+template <typename T>
+inline T* XList<T>::prev(T* elem) const {
+  verify_head();
+
+  XListNode<T>* const node = cast_to_inner(elem);
+  node->verify_links_linked();
+
+  XListNode<T>* const prev = node->_prev;
+  prev->verify_links_linked();
+
+  return (prev == &_head) ? NULL : cast_to_outer(prev);
+}
+
+template <typename T>
+inline void XList<T>::insert_first(T* elem) {
+  insert(&_head, cast_to_inner(elem));
+}
+
+template <typename T>
+inline void XList<T>::insert_last(T* elem) {
+  insert(_head._prev, cast_to_inner(elem));
+}
+
+template <typename T>
+inline void XList<T>::insert_before(T* before, T* elem) {
+  insert(cast_to_inner(before)->_prev, cast_to_inner(elem));
+}
+
+template <typename T>
+inline void XList<T>::insert_after(T* after, T* elem) {
+  insert(cast_to_inner(after), cast_to_inner(elem));
+}
+
+template <typename T>
+inline void XList<T>::remove(T* elem) {
+  verify_head();
+
+  XListNode<T>* const node = cast_to_inner(elem);
+  node->verify_links_linked();
+
+  XListNode<T>* const next = node->_next;
+  XListNode<T>* const prev = node->_prev;
+  next->verify_links_linked();
+  prev->verify_links_linked();
+
+  node->_next = prev->_next;
+  node->_prev = next->_prev;
+  node->verify_links_unlinked();
+
+  next->_prev = prev;
+  prev->_next = next;
+  next->verify_links();
+  prev->verify_links();
+
+  _size--;
+}
+
+template <typename T>
+inline T* XList<T>::remove_first() {
+  T* elem = first();
+  if (elem != NULL) {
+    remove(elem);
+  }
+
+  return elem;
+}
+
+template <typename T>
+inline T* XList<T>::remove_last() {
+  T* elem = last();
+  if (elem != NULL) {
+    remove(elem);
+  }
+
+  return elem;
+}
+
+template <typename T, bool Forward>
+inline XListIteratorImpl<T, Forward>::XListIteratorImpl(const XList<T>* list) :
+    _list(list),
+    _next(Forward ? list->first() : list->last()) {}
+
+template <typename T, bool Forward>
+inline bool XListIteratorImpl<T, Forward>::next(T** elem) {
+  if (_next != NULL) {
+    *elem = _next;
+    _next = Forward ? _list->next(_next) : _list->prev(_next);
+    return true;
+  }
+
+  // No more elements
+  return false;
+}
+
+template <typename T, bool Forward>
+inline XListRemoveIteratorImpl<T, Forward>::XListRemoveIteratorImpl(XList<T>* list) :
+    _list(list) {}
+
+template <typename T, bool Forward>
+inline bool XListRemoveIteratorImpl<T, Forward>::next(T** elem) {
+  *elem = Forward ? _list->remove_first() : _list->remove_last();
+  return *elem != NULL;
+}
+
+#endif // SHARE_GC_X_XLIST_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xLiveMap.cpp b/src/hotspot/share/gc/x/xLiveMap.cpp
new file mode 100644
index 00000000000..91ef99754f7
--- /dev/null
+++ b/src/hotspot/share/gc/x/xLiveMap.cpp
@@ -0,0 +1,133 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xHeap.inline.hpp"
+#include "gc/x/xLiveMap.inline.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xThread.inline.hpp"
+#include "logging/log.hpp"
+#include "runtime/atomic.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+static const XStatCounter XCounterMarkSeqNumResetContention("Contention", "Mark SeqNum Reset Contention", XStatUnitOpsPerSecond);
+static const XStatCounter XCounterMarkSegmentResetContention("Contention", "Mark Segment Reset Contention", XStatUnitOpsPerSecond);
+
+static size_t bitmap_size(uint32_t size, size_t nsegments) {
+  // We need at least one bit per segment
+  return MAX2<size_t>(size, nsegments) * 2;
+}
+
+XLiveMap::XLiveMap(uint32_t size) :
+    _seqnum(0),
+    _live_objects(0),
+    _live_bytes(0),
+    _segment_live_bits(0),
+    _segment_claim_bits(0),
+    _bitmap(bitmap_size(size, nsegments)),
+    _segment_shift(exact_log2(segment_size())) {}
+
+void XLiveMap::reset(size_t index) {
+  const uint32_t seqnum_initializing = (uint32_t)-1;
+  bool contention = false;
+
+  // Multiple threads can enter here, make sure only one of them
+  // resets the marking information while the others busy wait.
+  for (uint32_t seqnum = Atomic::load_acquire(&_seqnum);
+       seqnum != XGlobalSeqNum;
+       seqnum = Atomic::load_acquire(&_seqnum)) {
+    if ((seqnum != seqnum_initializing) &&
+        (Atomic::cmpxchg(&_seqnum, seqnum, seqnum_initializing) == seqnum)) {
+      // Reset marking information
+      _live_bytes = 0;
+      _live_objects = 0;
+
+      // Clear segment claimed/live bits
+      segment_live_bits().clear();
+      segment_claim_bits().clear();
+
+      assert(_seqnum == seqnum_initializing, "Invalid");
+
+      // Make sure the newly reset marking information is ordered
+      // before the update of the page seqnum, such that when the
+      // up-to-date seqnum is load acquired, the bit maps will not
+      // contain stale information.
+      Atomic::release_store(&_seqnum, XGlobalSeqNum);
+      break;
+    }
+
+    // Mark reset contention
+    if (!contention) {
+      // Count contention once
+      XStatInc(XCounterMarkSeqNumResetContention);
+      contention = true;
+
+      log_trace(gc)("Mark seqnum reset contention, thread: " PTR_FORMAT " (%s), map: " PTR_FORMAT ", bit: " SIZE_FORMAT,
+                    XThread::id(), XThread::name(), p2i(this), index);
+    }
+  }
+}
+
+void XLiveMap::reset_segment(BitMap::idx_t segment) {
+  bool contention = false;
+
+  if (!claim_segment(segment)) {
+    // Already claimed, wait for live bit to be set
+    while (!is_segment_live(segment)) {
+      // Mark reset contention
+      if (!contention) {
+        // Count contention once
+        XStatInc(XCounterMarkSegmentResetContention);
+        contention = true;
+
+        log_trace(gc)("Mark segment reset contention, thread: " PTR_FORMAT " (%s), map: " PTR_FORMAT ", segment: " SIZE_FORMAT,
+                      XThread::id(), XThread::name(), p2i(this), segment);
+      }
+    }
+
+    // Segment is live
+    return;
+  }
+
+  // Segment claimed, clear it
+  const BitMap::idx_t start_index = segment_start(segment);
+  const BitMap::idx_t end_index   = segment_end(segment);
+  if (segment_size() / BitsPerWord >= 32) {
+    _bitmap.clear_large_range(start_index, end_index);
+  } else {
+    _bitmap.clear_range(start_index, end_index);
+  }
+
+  // Set live bit
+  const bool success = set_segment_live(segment);
+  assert(success, "Should never fail");
+}
+
+void XLiveMap::resize(uint32_t size) {
+  const size_t new_bitmap_size = bitmap_size(size, nsegments);
+  if (_bitmap.size() != new_bitmap_size) {
+    _bitmap.reinitialize(new_bitmap_size, false /* clear */);
+    _segment_shift = exact_log2(segment_size());
+  }
+}
diff --git a/src/hotspot/share/gc/x/xLiveMap.hpp b/src/hotspot/share/gc/x/xLiveMap.hpp
new file mode 100644
index 00000000000..7bad774c6c6
--- /dev/null
+++ b/src/hotspot/share/gc/x/xLiveMap.hpp
@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XLIVEMAP_HPP
+#define SHARE_GC_X_XLIVEMAP_HPP
+
+#include "gc/x/xBitMap.hpp"
+#include "memory/allocation.hpp"
+
+class ObjectClosure;
+
+class XLiveMap {
+  friend class XLiveMapTest;
+
+private:
+  static const size_t nsegments = 64;
+
+  volatile uint32_t _seqnum;
+  volatile uint32_t _live_objects;
+  volatile size_t   _live_bytes;
+  BitMap::bm_word_t _segment_live_bits;
+  BitMap::bm_word_t _segment_claim_bits;
+  XBitMap           _bitmap;
+  size_t            _segment_shift;
+
+  const BitMapView segment_live_bits() const;
+  const BitMapView segment_claim_bits() const;
+
+  BitMapView segment_live_bits();
+  BitMapView segment_claim_bits();
+
+  BitMap::idx_t segment_size() const;
+
+  BitMap::idx_t segment_start(BitMap::idx_t segment) const;
+  BitMap::idx_t segment_end(BitMap::idx_t segment) const;
+
+  bool is_segment_live(BitMap::idx_t segment) const;
+  bool set_segment_live(BitMap::idx_t segment);
+
+  BitMap::idx_t first_live_segment() const;
+  BitMap::idx_t next_live_segment(BitMap::idx_t segment) const;
+  BitMap::idx_t index_to_segment(BitMap::idx_t index) const;
+
+  bool claim_segment(BitMap::idx_t segment);
+
+  void reset(size_t index);
+  void reset_segment(BitMap::idx_t segment);
+
+  void iterate_segment(ObjectClosure* cl, BitMap::idx_t segment, uintptr_t page_start, size_t page_object_alignment_shift);
+
+public:
+  XLiveMap(uint32_t size);
+
+  void reset();
+  void resize(uint32_t size);
+
+  bool is_marked() const;
+
+  uint32_t live_objects() const;
+  size_t live_bytes() const;
+
+  bool get(size_t index) const;
+  bool set(size_t index, bool finalizable, bool& inc_live);
+
+  void inc_live(uint32_t objects, size_t bytes);
+
+  void iterate(ObjectClosure* cl, uintptr_t page_start, size_t page_object_alignment_shift);
+};
+
+#endif // SHARE_GC_X_XLIVEMAP_HPP
diff --git a/src/hotspot/share/gc/x/xLiveMap.inline.hpp b/src/hotspot/share/gc/x/xLiveMap.inline.hpp
new file mode 100644
index 00000000000..f836f9ab4c2
--- /dev/null
+++ b/src/hotspot/share/gc/x/xLiveMap.inline.hpp
@@ -0,0 +1,175 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XLIVEMAP_INLINE_HPP
+#define SHARE_GC_X_XLIVEMAP_INLINE_HPP
+
+#include "gc/x/xLiveMap.hpp"
+
+#include "gc/x/xBitMap.inline.hpp"
+#include "gc/x/xMark.hpp"
+#include "gc/x/xOop.inline.hpp"
+#include "gc/x/xUtils.inline.hpp"
+#include "runtime/atomic.hpp"
+#include "utilities/bitMap.inline.hpp"
+#include "utilities/debug.hpp"
+
+inline void XLiveMap::reset() {
+  _seqnum = 0;
+}
+
+inline bool XLiveMap::is_marked() const {
+  return Atomic::load_acquire(&_seqnum) == XGlobalSeqNum;
+}
+
+inline uint32_t XLiveMap::live_objects() const {
+  assert(XGlobalPhase != XPhaseMark, "Invalid phase");
+  return _live_objects;
+}
+
+inline size_t XLiveMap::live_bytes() const {
+  assert(XGlobalPhase != XPhaseMark, "Invalid phase");
+  return _live_bytes;
+}
+
+inline const BitMapView XLiveMap::segment_live_bits() const {
+  return BitMapView(const_cast<BitMap::bm_word_t*>(&_segment_live_bits), nsegments);
+}
+
+inline const BitMapView XLiveMap::segment_claim_bits() const {
+  return BitMapView(const_cast<BitMap::bm_word_t*>(&_segment_claim_bits), nsegments);
+}
+
+inline BitMapView XLiveMap::segment_live_bits() {
+  return BitMapView(&_segment_live_bits, nsegments);
+}
+
+inline BitMapView XLiveMap::segment_claim_bits() {
+  return BitMapView(&_segment_claim_bits, nsegments);
+}
+
+inline bool XLiveMap::is_segment_live(BitMap::idx_t segment) const {
+  return segment_live_bits().par_at(segment);
+}
+
+inline bool XLiveMap::set_segment_live(BitMap::idx_t segment) {
+  return segment_live_bits().par_set_bit(segment, memory_order_release);
+}
+
+inline bool XLiveMap::claim_segment(BitMap::idx_t segment) {
+  return segment_claim_bits().par_set_bit(segment, memory_order_acq_rel);
+}
+
+inline BitMap::idx_t XLiveMap::first_live_segment() const {
+  return segment_live_bits().find_first_set_bit(0, nsegments);
+}
+
+inline BitMap::idx_t XLiveMap::next_live_segment(BitMap::idx_t segment) const {
+  return segment_live_bits().find_first_set_bit(segment + 1, nsegments);
+}
+
+inline BitMap::idx_t XLiveMap::segment_size() const {
+  return _bitmap.size() / nsegments;
+}
+
+inline BitMap::idx_t XLiveMap::index_to_segment(BitMap::idx_t index) const {
+  return index >> _segment_shift;
+}
+
+inline bool XLiveMap::get(size_t index) const {
+  BitMap::idx_t segment = index_to_segment(index);
+  return is_marked() &&                               // Page is marked
+         is_segment_live(segment) &&                  // Segment is marked
+         _bitmap.par_at(index, memory_order_relaxed); // Object is marked
+}
+
+inline bool XLiveMap::set(size_t index, bool finalizable, bool& inc_live) {
+  if (!is_marked()) {
+    // First object to be marked during this
+    // cycle, reset marking information.
+    reset(index);
+  }
+
+  const BitMap::idx_t segment = index_to_segment(index);
+  if (!is_segment_live(segment)) {
+    // First object to be marked in this segment during
+    // this cycle, reset segment bitmap.
+    reset_segment(segment);
+  }
+
+  return _bitmap.par_set_bit_pair(index, finalizable, inc_live);
+}
+
+inline void XLiveMap::inc_live(uint32_t objects, size_t bytes) {
+  Atomic::add(&_live_objects, objects);
+  Atomic::add(&_live_bytes, bytes);
+}
+
+inline BitMap::idx_t XLiveMap::segment_start(BitMap::idx_t segment) const {
+  return segment_size() * segment;
+}
+
+inline BitMap::idx_t XLiveMap::segment_end(BitMap::idx_t segment) const {
+  return segment_start(segment) + segment_size();
+}
+
+inline void XLiveMap::iterate_segment(ObjectClosure* cl, BitMap::idx_t segment, uintptr_t page_start, size_t page_object_alignment_shift) {
+  assert(is_segment_live(segment), "Must be");
+
+  const BitMap::idx_t start_index = segment_start(segment);
+  const BitMap::idx_t end_index   = segment_end(segment);
+  BitMap::idx_t index = _bitmap.find_first_set_bit(start_index, end_index);
+
+  while (index < end_index) {
+    // Calculate object address
+    const uintptr_t addr = page_start + ((index / 2) << page_object_alignment_shift);
+
+    // Get the size of the object before calling the closure, which
+    // might overwrite the object in case we are relocating in-place.
+    const size_t size = XUtils::object_size(addr);
+
+    // Apply closure
+    cl->do_object(XOop::from_address(addr));
+
+    // Find next bit after this object
+    const uintptr_t next_addr = align_up(addr + size, 1 << page_object_alignment_shift);
+    const BitMap::idx_t next_index = ((next_addr - page_start) >> page_object_alignment_shift) * 2;
+    if (next_index >= end_index) {
+      // End of live map
+      break;
+    }
+
+    index = _bitmap.find_first_set_bit(next_index, end_index);
+  }
+}
+
+inline void XLiveMap::iterate(ObjectClosure* cl, uintptr_t page_start, size_t page_object_alignment_shift) {
+  if (is_marked()) {
+    for (BitMap::idx_t segment = first_live_segment(); segment < nsegments; segment = next_live_segment(segment)) {
+      // For each live segment
+      iterate_segment(cl, segment, page_start, page_object_alignment_shift);
+    }
+  }
+}
+
+#endif // SHARE_GC_X_XLIVEMAP_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xLock.hpp b/src/hotspot/share/gc/x/xLock.hpp
new file mode 100644
index 00000000000..2ba612d033c
--- /dev/null
+++ b/src/hotspot/share/gc/x/xLock.hpp
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XLOCK_HPP
+#define SHARE_GC_X_XLOCK_HPP
+
+#include "memory/allocation.hpp"
+#include "runtime/mutex.hpp"
+
+class XLock {
+private:
+  PlatformMutex _lock;
+
+public:
+  void lock();
+  bool try_lock();
+  void unlock();
+};
+
+class XReentrantLock {
+private:
+  XLock            _lock;
+  Thread* volatile _owner;
+  uint64_t         _count;
+
+public:
+  XReentrantLock();
+
+  void lock();
+  void unlock();
+
+  bool is_owned() const;
+};
+
+class XConditionLock {
+private:
+  PlatformMonitor _lock;
+
+public:
+  void lock();
+  bool try_lock();
+  void unlock();
+
+  bool wait(uint64_t millis = 0);
+  void notify();
+  void notify_all();
+};
+
+template <typename T>
+class XLocker : public StackObj {
+private:
+  T* const _lock;
+
+public:
+  XLocker(T* lock);
+  ~XLocker();
+};
+
+#endif // SHARE_GC_X_XLOCK_HPP
diff --git a/src/hotspot/share/gc/x/xLock.inline.hpp b/src/hotspot/share/gc/x/xLock.inline.hpp
new file mode 100644
index 00000000000..07a673376a6
--- /dev/null
+++ b/src/hotspot/share/gc/x/xLock.inline.hpp
@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XLOCK_INLINE_HPP
+#define SHARE_GC_X_XLOCK_INLINE_HPP
+
+#include "gc/x/xLock.hpp"
+
+#include "runtime/atomic.hpp"
+#include "runtime/javaThread.hpp"
+#include "runtime/os.inline.hpp"
+#include "utilities/debug.hpp"
+
+inline void XLock::lock() {
+  _lock.lock();
+}
+
+inline bool XLock::try_lock() {
+  return _lock.try_lock();
+}
+
+inline void XLock::unlock() {
+  _lock.unlock();
+}
+
+inline XReentrantLock::XReentrantLock() :
+    _lock(),
+    _owner(NULL),
+    _count(0) {}
+
+inline void XReentrantLock::lock() {
+  Thread* const thread = Thread::current();
+  Thread* const owner = Atomic::load(&_owner);
+
+  if (owner != thread) {
+    _lock.lock();
+    Atomic::store(&_owner, thread);
+  }
+
+  _count++;
+}
+
+inline void XReentrantLock::unlock() {
+  assert(is_owned(), "Invalid owner");
+  assert(_count > 0, "Invalid count");
+
+  _count--;
+
+  if (_count == 0) {
+    Atomic::store(&_owner, (Thread*)NULL);
+    _lock.unlock();
+  }
+}
+
+inline bool XReentrantLock::is_owned() const {
+  Thread* const thread = Thread::current();
+  Thread* const owner = Atomic::load(&_owner);
+  return owner == thread;
+}
+
+inline void XConditionLock::lock() {
+  _lock.lock();
+}
+
+inline bool XConditionLock::try_lock() {
+  return _lock.try_lock();
+}
+
+inline void XConditionLock::unlock() {
+  _lock.unlock();
+}
+
+inline bool XConditionLock::wait(uint64_t millis) {
+  return _lock.wait(millis) == OS_OK;
+}
+
+inline void XConditionLock::notify() {
+  _lock.notify();
+}
+
+inline void XConditionLock::notify_all() {
+  _lock.notify_all();
+}
+
+template <typename T>
+inline XLocker<T>::XLocker(T* lock) :
+    _lock(lock) {
+  if (_lock != NULL) {
+    _lock->lock();
+  }
+}
+
+template <typename T>
+inline XLocker<T>::~XLocker() {
+  if (_lock != NULL) {
+    _lock->unlock();
+  }
+}
+
+#endif // SHARE_GC_X_XLOCK_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xMark.cpp b/src/hotspot/share/gc/x/xMark.cpp
new file mode 100644
index 00000000000..16574364ef9
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMark.cpp
@@ -0,0 +1,875 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "classfile/classLoaderData.hpp"
+#include "classfile/classLoaderDataGraph.hpp"
+#include "classfile/javaClasses.inline.hpp"
+#include "code/nmethod.hpp"
+#include "gc/shared/continuationGCSupport.inline.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/shared/stringdedup/stringDedup.hpp"
+#include "gc/shared/suspendibleThreadSet.hpp"
+#include "gc/x/xAbort.inline.hpp"
+#include "gc/x/xBarrier.inline.hpp"
+#include "gc/x/xHeap.inline.hpp"
+#include "gc/x/xLock.inline.hpp"
+#include "gc/x/xMark.inline.hpp"
+#include "gc/x/xMarkCache.inline.hpp"
+#include "gc/x/xMarkContext.inline.hpp"
+#include "gc/x/xMarkStack.inline.hpp"
+#include "gc/x/xMarkTerminate.inline.hpp"
+#include "gc/x/xNMethod.hpp"
+#include "gc/x/xOop.inline.hpp"
+#include "gc/x/xPage.hpp"
+#include "gc/x/xPageTable.inline.hpp"
+#include "gc/x/xRootsIterator.hpp"
+#include "gc/x/xStackWatermark.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xTask.hpp"
+#include "gc/x/xThread.inline.hpp"
+#include "gc/x/xThreadLocalAllocBuffer.hpp"
+#include "gc/x/xUtils.inline.hpp"
+#include "gc/x/xWorkers.hpp"
+#include "logging/log.hpp"
+#include "memory/iterator.inline.hpp"
+#include "oops/objArrayOop.inline.hpp"
+#include "oops/oop.inline.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/continuation.hpp"
+#include "runtime/handshake.hpp"
+#include "runtime/javaThread.hpp"
+#include "runtime/prefetch.inline.hpp"
+#include "runtime/safepointMechanism.hpp"
+#include "runtime/stackWatermark.hpp"
+#include "runtime/stackWatermarkSet.inline.hpp"
+#include "runtime/threads.hpp"
+#include "utilities/align.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/powerOfTwo.hpp"
+#include "utilities/ticks.hpp"
+
+static const XStatSubPhase XSubPhaseConcurrentMark("Concurrent Mark");
+static const XStatSubPhase XSubPhaseConcurrentMarkTryFlush("Concurrent Mark Try Flush");
+static const XStatSubPhase XSubPhaseConcurrentMarkTryTerminate("Concurrent Mark Try Terminate");
+static const XStatSubPhase XSubPhaseMarkTryComplete("Pause Mark Try Complete");
+
+XMark::XMark(XWorkers* workers, XPageTable* page_table) :
+    _workers(workers),
+    _page_table(page_table),
+    _allocator(),
+    _stripes(),
+    _terminate(),
+    _work_terminateflush(true),
+    _work_nproactiveflush(0),
+    _work_nterminateflush(0),
+    _nproactiveflush(0),
+    _nterminateflush(0),
+    _ntrycomplete(0),
+    _ncontinue(0),
+    _nworkers(0) {}
+
+bool XMark::is_initialized() const {
+  return _allocator.is_initialized();
+}
+
+size_t XMark::calculate_nstripes(uint nworkers) const {
+  // Calculate the number of stripes from the number of workers we use,
+  // where the number of stripes must be a power of two and we want to
+  // have at least one worker per stripe.
+  const size_t nstripes = round_down_power_of_2(nworkers);
+  return MIN2(nstripes, XMarkStripesMax);
+}
+
+void XMark::start() {
+  // Verification
+  if (ZVerifyMarking) {
+    verify_all_stacks_empty();
+  }
+
+  // Increment global sequence number to invalidate
+  // marking information for all pages.
+  XGlobalSeqNum++;
+
+  // Note that we start a marking cycle.
+  // Unlike other GCs, the color switch implicitly changes the nmethods
+  // to be armed, and the thread-local disarm values are lazily updated
+  // when JavaThreads wake up from safepoints.
+  CodeCache::on_gc_marking_cycle_start();
+
+  // Reset flush/continue counters
+  _nproactiveflush = 0;
+  _nterminateflush = 0;
+  _ntrycomplete = 0;
+  _ncontinue = 0;
+
+  // Set number of workers to use
+  _nworkers = _workers->active_workers();
+
+  // Set number of mark stripes to use, based on number
+  // of workers we will use in the concurrent mark phase.
+  const size_t nstripes = calculate_nstripes(_nworkers);
+  _stripes.set_nstripes(nstripes);
+
+  // Update statistics
+  XStatMark::set_at_mark_start(nstripes);
+
+  // Print worker/stripe distribution
+  LogTarget(Debug, gc, marking) log;
+  if (log.is_enabled()) {
+    log.print("Mark Worker/Stripe Distribution");
+    for (uint worker_id = 0; worker_id < _nworkers; worker_id++) {
+      const XMarkStripe* const stripe = _stripes.stripe_for_worker(_nworkers, worker_id);
+      const size_t stripe_id = _stripes.stripe_id(stripe);
+      log.print("  Worker %u(%u) -> Stripe " SIZE_FORMAT "(" SIZE_FORMAT ")",
+                worker_id, _nworkers, stripe_id, nstripes);
+    }
+  }
+}
+
+void XMark::prepare_work() {
+  assert(_nworkers == _workers->active_workers(), "Invalid number of workers");
+
+  // Set number of active workers
+  _terminate.reset(_nworkers);
+
+  // Reset flush counters
+  _work_nproactiveflush = _work_nterminateflush = 0;
+  _work_terminateflush = true;
+}
+
+void XMark::finish_work() {
+  // Accumulate proactive/terminate flush counters
+  _nproactiveflush += _work_nproactiveflush;
+  _nterminateflush += _work_nterminateflush;
+}
+
+bool XMark::is_array(uintptr_t addr) const {
+  return XOop::from_address(addr)->is_objArray();
+}
+
+void XMark::push_partial_array(uintptr_t addr, size_t size, bool finalizable) {
+  assert(is_aligned(addr, XMarkPartialArrayMinSize), "Address misaligned");
+  XMarkThreadLocalStacks* const stacks = XThreadLocalData::stacks(Thread::current());
+  XMarkStripe* const stripe = _stripes.stripe_for_addr(addr);
+  const uintptr_t offset = XAddress::offset(addr) >> XMarkPartialArrayMinSizeShift;
+  const uintptr_t length = size / oopSize;
+  const XMarkStackEntry entry(offset, length, finalizable);
+
+  log_develop_trace(gc, marking)("Array push partial: " PTR_FORMAT " (" SIZE_FORMAT "), stripe: " SIZE_FORMAT,
+                                 addr, size, _stripes.stripe_id(stripe));
+
+  stacks->push(&_allocator, &_stripes, stripe, entry, false /* publish */);
+}
+
+void XMark::follow_small_array(uintptr_t addr, size_t size, bool finalizable) {
+  assert(size <= XMarkPartialArrayMinSize, "Too large, should be split");
+  const size_t length = size / oopSize;
+
+  log_develop_trace(gc, marking)("Array follow small: " PTR_FORMAT " (" SIZE_FORMAT ")", addr, size);
+
+  XBarrier::mark_barrier_on_oop_array((oop*)addr, length, finalizable);
+}
+
+void XMark::follow_large_array(uintptr_t addr, size_t size, bool finalizable) {
+  assert(size <= (size_t)arrayOopDesc::max_array_length(T_OBJECT) * oopSize, "Too large");
+  assert(size > XMarkPartialArrayMinSize, "Too small, should not be split");
+  const uintptr_t start = addr;
+  const uintptr_t end = start + size;
+
+  // Calculate the aligned middle start/end/size, where the middle start
+  // should always be greater than the start (hence the +1 below) to make
+  // sure we always do some follow work, not just split the array into pieces.
+  const uintptr_t middle_start = align_up(start + 1, XMarkPartialArrayMinSize);
+  const size_t    middle_size = align_down(end - middle_start, XMarkPartialArrayMinSize);
+  const uintptr_t middle_end = middle_start + middle_size;
+
+  log_develop_trace(gc, marking)("Array follow large: " PTR_FORMAT "-" PTR_FORMAT" (" SIZE_FORMAT "), "
+                                 "middle: " PTR_FORMAT "-" PTR_FORMAT " (" SIZE_FORMAT ")",
+                                 start, end, size, middle_start, middle_end, middle_size);
+
+  // Push unaligned trailing part
+  if (end > middle_end) {
+    const uintptr_t trailing_addr = middle_end;
+    const size_t trailing_size = end - middle_end;
+    push_partial_array(trailing_addr, trailing_size, finalizable);
+  }
+
+  // Push aligned middle part(s)
+  uintptr_t partial_addr = middle_end;
+  while (partial_addr > middle_start) {
+    const size_t parts = 2;
+    const size_t partial_size = align_up((partial_addr - middle_start) / parts, XMarkPartialArrayMinSize);
+    partial_addr -= partial_size;
+    push_partial_array(partial_addr, partial_size, finalizable);
+  }
+
+  // Follow leading part
+  assert(start < middle_start, "Miscalculated middle start");
+  const uintptr_t leading_addr = start;
+  const size_t leading_size = middle_start - start;
+  follow_small_array(leading_addr, leading_size, finalizable);
+}
+
+void XMark::follow_array(uintptr_t addr, size_t size, bool finalizable) {
+  if (size <= XMarkPartialArrayMinSize) {
+    follow_small_array(addr, size, finalizable);
+  } else {
+    follow_large_array(addr, size, finalizable);
+  }
+}
+
+void XMark::follow_partial_array(XMarkStackEntry entry, bool finalizable) {
+  const uintptr_t addr = XAddress::good(entry.partial_array_offset() << XMarkPartialArrayMinSizeShift);
+  const size_t size = entry.partial_array_length() * oopSize;
+
+  follow_array(addr, size, finalizable);
+}
+
+template <bool finalizable>
+class XMarkBarrierOopClosure : public ClaimMetadataVisitingOopIterateClosure {
+public:
+  XMarkBarrierOopClosure() :
+      ClaimMetadataVisitingOopIterateClosure(finalizable
+                                                 ? ClassLoaderData::_claim_finalizable
+                                                 : ClassLoaderData::_claim_strong,
+                                             finalizable
+                                                 ? NULL
+                                                 : XHeap::heap()->reference_discoverer()) {}
+
+  virtual void do_oop(oop* p) {
+    XBarrier::mark_barrier_on_oop_field(p, finalizable);
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+
+  virtual void do_nmethod(nmethod* nm) {
+    assert(!finalizable, "Can't handle finalizable marking of nmethods");
+    nm->run_nmethod_entry_barrier();
+  }
+};
+
+void XMark::follow_array_object(objArrayOop obj, bool finalizable) {
+  if (finalizable) {
+    XMarkBarrierOopClosure<true /* finalizable */> cl;
+    cl.do_klass(obj->klass());
+  } else {
+    XMarkBarrierOopClosure<false /* finalizable */> cl;
+    cl.do_klass(obj->klass());
+  }
+
+  const uintptr_t addr = (uintptr_t)obj->base();
+  const size_t size = (size_t)obj->length() * oopSize;
+
+  follow_array(addr, size, finalizable);
+}
+
+void XMark::follow_object(oop obj, bool finalizable) {
+  if (ContinuationGCSupport::relativize_stack_chunk(obj)) {
+    // Loom doesn't support mixing of finalizable marking and strong marking of
+    // stack chunks. See: RelativizeDerivedOopClosure.
+    XMarkBarrierOopClosure<false /* finalizable */> cl;
+    obj->oop_iterate(&cl);
+    return;
+  }
+
+  if (finalizable) {
+    XMarkBarrierOopClosure<true /* finalizable */> cl;
+    obj->oop_iterate(&cl);
+  } else {
+    XMarkBarrierOopClosure<false /* finalizable */> cl;
+    obj->oop_iterate(&cl);
+  }
+}
+
+static void try_deduplicate(XMarkContext* context, oop obj) {
+  if (!StringDedup::is_enabled()) {
+    // Not enabled
+    return;
+  }
+
+  if (!java_lang_String::is_instance(obj)) {
+    // Not a String object
+    return;
+  }
+
+  if (java_lang_String::test_and_set_deduplication_requested(obj)) {
+    // Already requested deduplication
+    return;
+  }
+
+  // Request deduplication
+  context->string_dedup_requests()->add(obj);
+}
+
+void XMark::mark_and_follow(XMarkContext* context, XMarkStackEntry entry) {
+  // Decode flags
+  const bool finalizable = entry.finalizable();
+  const bool partial_array = entry.partial_array();
+
+  if (partial_array) {
+    follow_partial_array(entry, finalizable);
+    return;
+  }
+
+  // Decode object address and additional flags
+  const uintptr_t addr = entry.object_address();
+  const bool mark = entry.mark();
+  bool inc_live = entry.inc_live();
+  const bool follow = entry.follow();
+
+  XPage* const page = _page_table->get(addr);
+  assert(page->is_relocatable(), "Invalid page state");
+
+  // Mark
+  if (mark && !page->mark_object(addr, finalizable, inc_live)) {
+    // Already marked
+    return;
+  }
+
+  // Increment live
+  if (inc_live) {
+    // Update live objects/bytes for page. We use the aligned object
+    // size since that is the actual number of bytes used on the page
+    // and alignment paddings can never be reclaimed.
+    const size_t size = XUtils::object_size(addr);
+    const size_t aligned_size = align_up(size, page->object_alignment());
+    context->cache()->inc_live(page, aligned_size);
+  }
+
+  // Follow
+  if (follow) {
+    if (is_array(addr)) {
+      follow_array_object(objArrayOop(XOop::from_address(addr)), finalizable);
+    } else {
+      const oop obj = XOop::from_address(addr);
+      follow_object(obj, finalizable);
+
+      // Try deduplicate
+      try_deduplicate(context, obj);
+    }
+  }
+}
+
+template <typename T>
+bool XMark::drain(XMarkContext* context, T* timeout) {
+  XMarkStripe* const stripe = context->stripe();
+  XMarkThreadLocalStacks* const stacks = context->stacks();
+  XMarkStackEntry entry;
+
+  // Drain stripe stacks
+  while (stacks->pop(&_allocator, &_stripes, stripe, entry)) {
+    mark_and_follow(context, entry);
+
+    // Check timeout
+    if (timeout->has_expired()) {
+      // Timeout
+      return false;
+    }
+  }
+
+  // Success
+  return !timeout->has_expired();
+}
+
+bool XMark::try_steal_local(XMarkContext* context) {
+  XMarkStripe* const stripe = context->stripe();
+  XMarkThreadLocalStacks* const stacks = context->stacks();
+
+  // Try to steal a local stack from another stripe
+  for (XMarkStripe* victim_stripe = _stripes.stripe_next(stripe);
+       victim_stripe != stripe;
+       victim_stripe = _stripes.stripe_next(victim_stripe)) {
+    XMarkStack* const stack = stacks->steal(&_stripes, victim_stripe);
+    if (stack != NULL) {
+      // Success, install the stolen stack
+      stacks->install(&_stripes, stripe, stack);
+      return true;
+    }
+  }
+
+  // Nothing to steal
+  return false;
+}
+
+bool XMark::try_steal_global(XMarkContext* context) {
+  XMarkStripe* const stripe = context->stripe();
+  XMarkThreadLocalStacks* const stacks = context->stacks();
+
+  // Try to steal a stack from another stripe
+  for (XMarkStripe* victim_stripe = _stripes.stripe_next(stripe);
+       victim_stripe != stripe;
+       victim_stripe = _stripes.stripe_next(victim_stripe)) {
+    XMarkStack* const stack = victim_stripe->steal_stack();
+    if (stack != NULL) {
+      // Success, install the stolen stack
+      stacks->install(&_stripes, stripe, stack);
+      return true;
+    }
+  }
+
+  // Nothing to steal
+  return false;
+}
+
+bool XMark::try_steal(XMarkContext* context) {
+  return try_steal_local(context) || try_steal_global(context);
+}
+
+void XMark::idle() const {
+  os::naked_short_sleep(1);
+}
+
+class XMarkFlushAndFreeStacksClosure : public HandshakeClosure {
+private:
+  XMark* const _mark;
+  bool         _flushed;
+
+public:
+  XMarkFlushAndFreeStacksClosure(XMark* mark) :
+      HandshakeClosure("XMarkFlushAndFreeStacks"),
+      _mark(mark),
+      _flushed(false) {}
+
+  void do_thread(Thread* thread) {
+    if (_mark->flush_and_free(thread)) {
+      _flushed = true;
+    }
+  }
+
+  bool flushed() const {
+    return _flushed;
+  }
+};
+
+bool XMark::flush(bool at_safepoint) {
+  XMarkFlushAndFreeStacksClosure cl(this);
+  if (at_safepoint) {
+    Threads::threads_do(&cl);
+  } else {
+    Handshake::execute(&cl);
+  }
+
+  // Returns true if more work is available
+  return cl.flushed() || !_stripes.is_empty();
+}
+
+bool XMark::try_flush(volatile size_t* nflush) {
+  Atomic::inc(nflush);
+
+  XStatTimer timer(XSubPhaseConcurrentMarkTryFlush);
+  return flush(false /* at_safepoint */);
+}
+
+bool XMark::try_proactive_flush() {
+  // Only do proactive flushes from worker 0
+  if (XThread::worker_id() != 0) {
+    return false;
+  }
+
+  if (Atomic::load(&_work_nproactiveflush) == XMarkProactiveFlushMax ||
+      Atomic::load(&_work_nterminateflush) != 0) {
+    // Limit reached or we're trying to terminate
+    return false;
+  }
+
+  return try_flush(&_work_nproactiveflush);
+}
+
+bool XMark::try_terminate() {
+  XStatTimer timer(XSubPhaseConcurrentMarkTryTerminate);
+
+  if (_terminate.enter_stage0()) {
+    // Last thread entered stage 0, flush
+    if (Atomic::load(&_work_terminateflush) &&
+        Atomic::load(&_work_nterminateflush) != XMarkTerminateFlushMax) {
+      // Exit stage 0 to allow other threads to continue marking
+      _terminate.exit_stage0();
+
+      // Flush before termination
+      if (!try_flush(&_work_nterminateflush)) {
+        // No more work available, skip further flush attempts
+        Atomic::store(&_work_terminateflush, false);
+      }
+
+      // Don't terminate, regardless of whether we successfully
+      // flushed out more work or not. We've already exited
+      // termination stage 0, to allow other threads to continue
+      // marking, so this thread has to return false and also
+      // make another round of attempted marking.
+      return false;
+    }
+  }
+
+  for (;;) {
+    if (_terminate.enter_stage1()) {
+      // Last thread entered stage 1, terminate
+      return true;
+    }
+
+    // Idle to give the other threads
+    // a chance to enter termination.
+    idle();
+
+    if (!_terminate.try_exit_stage1()) {
+      // All workers in stage 1, terminate
+      return true;
+    }
+
+    if (_terminate.try_exit_stage0()) {
+      // More work available, don't terminate
+      return false;
+    }
+  }
+}
+
+class XMarkNoTimeout : public StackObj {
+public:
+  bool has_expired() {
+    // No timeout, but check for signal to abort
+    return XAbort::should_abort();
+  }
+};
+
+void XMark::work_without_timeout(XMarkContext* context) {
+  XStatTimer timer(XSubPhaseConcurrentMark);
+  XMarkNoTimeout no_timeout;
+
+  for (;;) {
+    if (!drain(context, &no_timeout)) {
+      // Abort
+      break;
+    }
+
+    if (try_steal(context)) {
+      // Stole work
+      continue;
+    }
+
+    if (try_proactive_flush()) {
+      // Work available
+      continue;
+    }
+
+    if (try_terminate()) {
+      // Terminate
+      break;
+    }
+  }
+}
+
+class XMarkTimeout : public StackObj {
+private:
+  const Ticks    _start;
+  const uint64_t _timeout;
+  const uint64_t _check_interval;
+  uint64_t       _check_at;
+  uint64_t       _check_count;
+  bool           _expired;
+
+public:
+  XMarkTimeout(uint64_t timeout_in_micros) :
+      _start(Ticks::now()),
+      _timeout(_start.value() + TimeHelper::micros_to_counter(timeout_in_micros)),
+      _check_interval(200),
+      _check_at(_check_interval),
+      _check_count(0),
+      _expired(false) {}
+
+  ~XMarkTimeout() {
+    const Tickspan duration = Ticks::now() - _start;
+    log_debug(gc, marking)("Mark With Timeout (%s): %s, " UINT64_FORMAT " oops, %.3fms",
+                           XThread::name(), _expired ? "Expired" : "Completed",
+                           _check_count, TimeHelper::counter_to_millis(duration.value()));
+  }
+
+  bool has_expired() {
+    if (++_check_count == _check_at) {
+      _check_at += _check_interval;
+      if ((uint64_t)Ticks::now().value() >= _timeout) {
+        // Timeout
+        _expired = true;
+      }
+    }
+
+    return _expired;
+  }
+};
+
+void XMark::work_with_timeout(XMarkContext* context, uint64_t timeout_in_micros) {
+  XStatTimer timer(XSubPhaseMarkTryComplete);
+  XMarkTimeout timeout(timeout_in_micros);
+
+  for (;;) {
+    if (!drain(context, &timeout)) {
+      // Timed out
+      break;
+    }
+
+    if (try_steal(context)) {
+      // Stole work
+      continue;
+    }
+
+    // Terminate
+    break;
+  }
+}
+
+void XMark::work(uint64_t timeout_in_micros) {
+  XMarkStripe* const stripe = _stripes.stripe_for_worker(_nworkers, XThread::worker_id());
+  XMarkThreadLocalStacks* const stacks = XThreadLocalData::stacks(Thread::current());
+  XMarkContext context(_stripes.nstripes(), stripe, stacks);
+
+  if (timeout_in_micros == 0) {
+    work_without_timeout(&context);
+  } else {
+    work_with_timeout(&context, timeout_in_micros);
+  }
+
+  // Flush and publish stacks
+  stacks->flush(&_allocator, &_stripes);
+
+  // Free remaining stacks
+  stacks->free(&_allocator);
+}
+
+class XMarkOopClosure : public OopClosure {
+  virtual void do_oop(oop* p) {
+    XBarrier::mark_barrier_on_oop_field(p, false /* finalizable */);
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+};
+
+class XMarkThreadClosure : public ThreadClosure {
+private:
+  OopClosure* const _cl;
+
+public:
+  XMarkThreadClosure(OopClosure* cl) :
+      _cl(cl) {
+    XThreadLocalAllocBuffer::reset_statistics();
+  }
+  ~XMarkThreadClosure() {
+    XThreadLocalAllocBuffer::publish_statistics();
+  }
+  virtual void do_thread(Thread* thread) {
+    JavaThread* const jt = JavaThread::cast(thread);
+    StackWatermarkSet::finish_processing(jt, _cl, StackWatermarkKind::gc);
+    XThreadLocalAllocBuffer::update_stats(jt);
+  }
+};
+
+class XMarkNMethodClosure : public NMethodClosure {
+private:
+  OopClosure* const _cl;
+
+public:
+  XMarkNMethodClosure(OopClosure* cl) :
+      _cl(cl) {}
+
+  virtual void do_nmethod(nmethod* nm) {
+    XLocker<XReentrantLock> locker(XNMethod::lock_for_nmethod(nm));
+    if (XNMethod::is_armed(nm)) {
+      XNMethod::nmethod_oops_do_inner(nm, _cl);
+
+      // CodeCache unloading support
+      nm->mark_as_maybe_on_stack();
+
+      XNMethod::disarm(nm);
+    }
+  }
+};
+
+typedef ClaimingCLDToOopClosure<ClassLoaderData::_claim_strong> XMarkCLDClosure;
+
+class XMarkRootsTask : public XTask {
+private:
+  XMark* const               _mark;
+  SuspendibleThreadSetJoiner _sts_joiner;
+  XRootsIterator             _roots;
+
+  XMarkOopClosure            _cl;
+  XMarkCLDClosure            _cld_cl;
+  XMarkThreadClosure         _thread_cl;
+  XMarkNMethodClosure        _nm_cl;
+
+public:
+  XMarkRootsTask(XMark* mark) :
+      XTask("XMarkRootsTask"),
+      _mark(mark),
+      _sts_joiner(),
+      _roots(ClassLoaderData::_claim_strong),
+      _cl(),
+      _cld_cl(&_cl),
+      _thread_cl(&_cl),
+      _nm_cl(&_cl) {
+    ClassLoaderDataGraph_lock->lock();
+  }
+
+  ~XMarkRootsTask() {
+    ClassLoaderDataGraph_lock->unlock();
+  }
+
+  virtual void work() {
+    _roots.apply(&_cl,
+                 &_cld_cl,
+                 &_thread_cl,
+                 &_nm_cl);
+
+    // Flush and free worker stacks. Needed here since
+    // the set of workers executing during root scanning
+    // can be different from the set of workers executing
+    // during mark.
+    _mark->flush_and_free();
+  }
+};
+
+class XMarkTask : public XTask {
+private:
+  XMark* const   _mark;
+  const uint64_t _timeout_in_micros;
+
+public:
+  XMarkTask(XMark* mark, uint64_t timeout_in_micros = 0) :
+      XTask("XMarkTask"),
+      _mark(mark),
+      _timeout_in_micros(timeout_in_micros) {
+    _mark->prepare_work();
+  }
+
+  ~XMarkTask() {
+    _mark->finish_work();
+  }
+
+  virtual void work() {
+    _mark->work(_timeout_in_micros);
+  }
+};
+
+void XMark::mark(bool initial) {
+  if (initial) {
+    XMarkRootsTask task(this);
+    _workers->run(&task);
+  }
+
+  XMarkTask task(this);
+  _workers->run(&task);
+}
+
+bool XMark::try_complete() {
+  _ntrycomplete++;
+
+  // Use nconcurrent number of worker threads to maintain the
+  // worker/stripe distribution used during concurrent mark.
+  XMarkTask task(this, XMarkCompleteTimeout);
+  _workers->run(&task);
+
+  // Successful if all stripes are empty
+  return _stripes.is_empty();
+}
+
+bool XMark::try_end() {
+  // Flush all mark stacks
+  if (!flush(true /* at_safepoint */)) {
+    // Mark completed
+    return true;
+  }
+
+  // Try complete marking by doing a limited
+  // amount of mark work in this phase.
+  return try_complete();
+}
+
+bool XMark::end() {
+  // Try end marking
+  if (!try_end()) {
+    // Mark not completed
+    _ncontinue++;
+    return false;
+  }
+
+  // Verification
+  if (ZVerifyMarking) {
+    verify_all_stacks_empty();
+  }
+
+  // Update statistics
+  XStatMark::set_at_mark_end(_nproactiveflush, _nterminateflush, _ntrycomplete, _ncontinue);
+
+  // Note that we finished a marking cycle.
+  // Unlike other GCs, we do not arm the nmethods
+  // when marking terminates.
+  CodeCache::on_gc_marking_cycle_finish();
+
+  // Mark completed
+  return true;
+}
+
+void XMark::free() {
+  // Free any unused mark stack space
+  _allocator.free();
+
+  // Update statistics
+  XStatMark::set_at_mark_free(_allocator.size());
+}
+
+void XMark::flush_and_free() {
+  Thread* const thread = Thread::current();
+  flush_and_free(thread);
+}
+
+bool XMark::flush_and_free(Thread* thread) {
+  XMarkThreadLocalStacks* const stacks = XThreadLocalData::stacks(thread);
+  const bool flushed = stacks->flush(&_allocator, &_stripes);
+  stacks->free(&_allocator);
+  return flushed;
+}
+
+class XVerifyMarkStacksEmptyClosure : public ThreadClosure {
+private:
+  const XMarkStripeSet* const _stripes;
+
+public:
+  XVerifyMarkStacksEmptyClosure(const XMarkStripeSet* stripes) :
+      _stripes(stripes) {}
+
+  void do_thread(Thread* thread) {
+    XMarkThreadLocalStacks* const stacks = XThreadLocalData::stacks(thread);
+    guarantee(stacks->is_empty(_stripes), "Should be empty");
+  }
+};
+
+void XMark::verify_all_stacks_empty() const {
+  // Verify thread stacks
+  XVerifyMarkStacksEmptyClosure cl(&_stripes);
+  Threads::threads_do(&cl);
+
+  // Verify stripe stacks
+  guarantee(_stripes.is_empty(), "Should be empty");
+}
diff --git a/src/hotspot/share/gc/x/xMark.hpp b/src/hotspot/share/gc/x/xMark.hpp
new file mode 100644
index 00000000000..5e40b79f02e
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMark.hpp
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XMARK_HPP
+#define SHARE_GC_X_XMARK_HPP
+
+#include "gc/x/xMarkStack.hpp"
+#include "gc/x/xMarkStackAllocator.hpp"
+#include "gc/x/xMarkStackEntry.hpp"
+#include "gc/x/xMarkTerminate.hpp"
+#include "oops/oopsHierarchy.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class Thread;
+class XMarkContext;
+class XPageTable;
+class XWorkers;
+
+class XMark {
+  friend class XMarkTask;
+
+private:
+  XWorkers* const     _workers;
+  XPageTable* const   _page_table;
+  XMarkStackAllocator _allocator;
+  XMarkStripeSet      _stripes;
+  XMarkTerminate      _terminate;
+  volatile bool       _work_terminateflush;
+  volatile size_t     _work_nproactiveflush;
+  volatile size_t     _work_nterminateflush;
+  size_t              _nproactiveflush;
+  size_t              _nterminateflush;
+  size_t              _ntrycomplete;
+  size_t              _ncontinue;
+  uint                _nworkers;
+
+  size_t calculate_nstripes(uint nworkers) const;
+
+  bool is_array(uintptr_t addr) const;
+  void push_partial_array(uintptr_t addr, size_t size, bool finalizable);
+  void follow_small_array(uintptr_t addr, size_t size, bool finalizable);
+  void follow_large_array(uintptr_t addr, size_t size, bool finalizable);
+  void follow_array(uintptr_t addr, size_t size, bool finalizable);
+  void follow_partial_array(XMarkStackEntry entry, bool finalizable);
+  void follow_array_object(objArrayOop obj, bool finalizable);
+  void follow_object(oop obj, bool finalizable);
+  void mark_and_follow(XMarkContext* context, XMarkStackEntry entry);
+
+  template <typename T> bool drain(XMarkContext* context, T* timeout);
+  bool try_steal_local(XMarkContext* context);
+  bool try_steal_global(XMarkContext* context);
+  bool try_steal(XMarkContext* context);
+  void idle() const;
+  bool flush(bool at_safepoint);
+  bool try_proactive_flush();
+  bool try_flush(volatile size_t* nflush);
+  bool try_terminate();
+  bool try_complete();
+  bool try_end();
+
+  void prepare_work();
+  void finish_work();
+
+  void work_without_timeout(XMarkContext* context);
+  void work_with_timeout(XMarkContext* context, uint64_t timeout_in_micros);
+  void work(uint64_t timeout_in_micros);
+
+  void verify_all_stacks_empty() const;
+
+public:
+  XMark(XWorkers* workers, XPageTable* page_table);
+
+  bool is_initialized() const;
+
+  template <bool gc_thread, bool follow, bool finalizable, bool publish> void mark_object(uintptr_t addr);
+
+  void start();
+  void mark(bool initial);
+  bool end();
+  void free();
+
+  void flush_and_free();
+  bool flush_and_free(Thread* thread);
+};
+
+#endif // SHARE_GC_X_XMARK_HPP
diff --git a/src/hotspot/share/gc/x/xMark.inline.hpp b/src/hotspot/share/gc/x/xMark.inline.hpp
new file mode 100644
index 00000000000..1f8fc81f525
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMark.inline.hpp
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XMARK_INLINE_HPP
+#define SHARE_GC_X_XMARK_INLINE_HPP
+
+#include "gc/x/xMark.hpp"
+
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xMarkStack.inline.hpp"
+#include "gc/x/xPage.inline.hpp"
+#include "gc/x/xPageTable.inline.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+#include "runtime/javaThread.hpp"
+#include "utilities/debug.hpp"
+
+// Marking before pushing helps reduce mark stack memory usage. However,
+// we only mark before pushing in GC threads to avoid burdening Java threads
+// with writing to, and potentially first having to clear, mark bitmaps.
+//
+// It's also worth noting that while marking an object can be done at any
+// time in the marking phase, following an object can only be done after
+// root processing has called ClassLoaderDataGraph::clear_claimed_marks(),
+// since it otherwise would interact badly with claiming of CLDs.
+
+template <bool gc_thread, bool follow, bool finalizable, bool publish>
+inline void XMark::mark_object(uintptr_t addr) {
+  assert(XAddress::is_marked(addr), "Should be marked");
+
+  XPage* const page = _page_table->get(addr);
+  if (page->is_allocating()) {
+    // Already implicitly marked
+    return;
+  }
+
+  const bool mark_before_push = gc_thread;
+  bool inc_live = false;
+
+  if (mark_before_push) {
+    // Try mark object
+    if (!page->mark_object(addr, finalizable, inc_live)) {
+      // Already marked
+      return;
+    }
+  } else {
+    // Don't push if already marked
+    if (page->is_object_marked<finalizable>(addr)) {
+      // Already marked
+      return;
+    }
+  }
+
+  // Push
+  XMarkThreadLocalStacks* const stacks = XThreadLocalData::stacks(Thread::current());
+  XMarkStripe* const stripe = _stripes.stripe_for_addr(addr);
+  XMarkStackEntry entry(addr, !mark_before_push, inc_live, follow, finalizable);
+  stacks->push(&_allocator, &_stripes, stripe, entry, publish);
+}
+
+#endif // SHARE_GC_X_XMARK_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xMarkCache.cpp b/src/hotspot/share/gc/x/xMarkCache.cpp
new file mode 100644
index 00000000000..bb70683221c
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMarkCache.cpp
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xMarkCache.inline.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+XMarkCacheEntry::XMarkCacheEntry() :
+    _page(NULL),
+    _objects(0),
+    _bytes(0) {}
+
+XMarkCache::XMarkCache(size_t nstripes) :
+    _shift(XMarkStripeShift + exact_log2(nstripes)) {}
+
+XMarkCache::~XMarkCache() {
+  // Evict all entries
+  for (size_t i = 0; i < XMarkCacheSize; i++) {
+    _cache[i].evict();
+  }
+}
+
diff --git a/src/hotspot/share/gc/x/xMarkCache.hpp b/src/hotspot/share/gc/x/xMarkCache.hpp
new file mode 100644
index 00000000000..8fbdc873522
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMarkCache.hpp
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XMARKCACHE_HPP
+#define SHARE_GC_X_XMARKCACHE_HPP
+
+#include "gc/x/xGlobals.hpp"
+#include "memory/allocation.hpp"
+
+class XPage;
+
+class XMarkCacheEntry {
+private:
+  XPage*   _page;
+  uint32_t _objects;
+  size_t   _bytes;
+
+public:
+  XMarkCacheEntry();
+
+  void inc_live(XPage* page, size_t bytes);
+  void evict();
+};
+
+class XMarkCache : public StackObj {
+private:
+  const size_t    _shift;
+  XMarkCacheEntry _cache[XMarkCacheSize];
+
+public:
+  XMarkCache(size_t nstripes);
+  ~XMarkCache();
+
+  void inc_live(XPage* page, size_t bytes);
+};
+
+#endif // SHARE_GC_X_XMARKCACHE_HPP
diff --git a/src/hotspot/share/gc/x/xMarkCache.inline.hpp b/src/hotspot/share/gc/x/xMarkCache.inline.hpp
new file mode 100644
index 00000000000..8eaf04a68fe
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMarkCache.inline.hpp
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XMARKCACHE_INLINE_HPP
+#define SHARE_GC_X_XMARKCACHE_INLINE_HPP
+
+#include "gc/x/xMarkCache.hpp"
+
+#include "gc/x/xPage.inline.hpp"
+
+inline void XMarkCacheEntry::inc_live(XPage* page, size_t bytes) {
+  if (_page == page) {
+    // Cache hit
+    _objects++;
+    _bytes += bytes;
+  } else {
+    // Cache miss
+    evict();
+    _page = page;
+    _objects = 1;
+    _bytes = bytes;
+  }
+}
+
+inline void XMarkCacheEntry::evict() {
+  if (_page != NULL) {
+    // Write cached data out to page
+    _page->inc_live(_objects, _bytes);
+    _page = NULL;
+  }
+}
+
+inline void XMarkCache::inc_live(XPage* page, size_t bytes) {
+  const size_t mask = XMarkCacheSize - 1;
+  const size_t index = (page->start() >> _shift) & mask;
+  _cache[index].inc_live(page, bytes);
+}
+
+#endif // SHARE_GC_X_XMARKCACHE_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xMarkContext.hpp b/src/hotspot/share/gc/x/xMarkContext.hpp
new file mode 100644
index 00000000000..246822931b7
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMarkContext.hpp
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XMARKCONTEXT_HPP
+#define SHARE_GC_X_XMARKCONTEXT_HPP
+
+#include "gc/x/xMarkCache.hpp"
+#include "gc/shared/stringdedup/stringDedup.hpp"
+#include "memory/allocation.hpp"
+
+class XMarkStripe;
+class XMarkThreadLocalStacks;
+
+class XMarkContext : public StackObj {
+private:
+  XMarkCache                    _cache;
+  XMarkStripe* const            _stripe;
+  XMarkThreadLocalStacks* const _stacks;
+  StringDedup::Requests         _string_dedup_requests;
+
+public:
+  XMarkContext(size_t nstripes,
+               XMarkStripe* stripe,
+               XMarkThreadLocalStacks* stacks);
+
+  XMarkCache* cache();
+  XMarkStripe* stripe();
+  XMarkThreadLocalStacks* stacks();
+  StringDedup::Requests* string_dedup_requests();
+};
+
+#endif // SHARE_GC_X_XMARKCONTEXT_HPP
diff --git a/src/hotspot/share/gc/x/xMarkContext.inline.hpp b/src/hotspot/share/gc/x/xMarkContext.inline.hpp
new file mode 100644
index 00000000000..74a182b67c3
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMarkContext.inline.hpp
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XMARKCONTEXT_INLINE_HPP
+#define SHARE_GC_X_XMARKCONTEXT_INLINE_HPP
+
+#include "gc/x/xMarkContext.hpp"
+
+inline XMarkContext::XMarkContext(size_t nstripes,
+                                  XMarkStripe* stripe,
+                                  XMarkThreadLocalStacks* stacks) :
+    _cache(nstripes),
+    _stripe(stripe),
+    _stacks(stacks),
+    _string_dedup_requests() {}
+
+inline XMarkCache* XMarkContext::cache() {
+  return &_cache;
+}
+
+inline XMarkStripe* XMarkContext::stripe() {
+  return _stripe;
+}
+
+inline XMarkThreadLocalStacks* XMarkContext::stacks() {
+  return _stacks;
+}
+
+inline StringDedup::Requests* XMarkContext::string_dedup_requests() {
+  return &_string_dedup_requests;
+}
+
+#endif // SHARE_GC_X_XMARKCACHE_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xMarkStack.cpp b/src/hotspot/share/gc/x/xMarkStack.cpp
new file mode 100644
index 00000000000..384dc200a95
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMarkStack.cpp
@@ -0,0 +1,226 @@
+/*
+ * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xMarkStack.inline.hpp"
+#include "gc/x/xMarkStackAllocator.hpp"
+#include "logging/log.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+XMarkStripe::XMarkStripe() :
+    _published(),
+    _overflowed() {}
+
+XMarkStripeSet::XMarkStripeSet() :
+    _nstripes(0),
+    _nstripes_mask(0),
+    _stripes() {}
+
+void XMarkStripeSet::set_nstripes(size_t nstripes) {
+  assert(is_power_of_2(nstripes), "Must be a power of two");
+  assert(is_power_of_2(XMarkStripesMax), "Must be a power of two");
+  assert(nstripes >= 1, "Invalid number of stripes");
+  assert(nstripes <= XMarkStripesMax, "Invalid number of stripes");
+
+  _nstripes = nstripes;
+  _nstripes_mask = nstripes - 1;
+
+  log_debug(gc, marking)("Using " SIZE_FORMAT " mark stripes", _nstripes);
+}
+
+bool XMarkStripeSet::is_empty() const {
+  for (size_t i = 0; i < _nstripes; i++) {
+    if (!_stripes[i].is_empty()) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+XMarkStripe* XMarkStripeSet::stripe_for_worker(uint nworkers, uint worker_id) {
+  const size_t spillover_limit = (nworkers / _nstripes) * _nstripes;
+  size_t index;
+
+  if (worker_id < spillover_limit) {
+    // Not a spillover worker, use natural stripe
+    index = worker_id & _nstripes_mask;
+  } else {
+    // Distribute spillover workers evenly across stripes
+    const size_t spillover_nworkers = nworkers - spillover_limit;
+    const size_t spillover_worker_id = worker_id - spillover_limit;
+    const double spillover_chunk = (double)_nstripes / (double)spillover_nworkers;
+    index = spillover_worker_id * spillover_chunk;
+  }
+
+  assert(index < _nstripes, "Invalid index");
+  return &_stripes[index];
+}
+
+XMarkThreadLocalStacks::XMarkThreadLocalStacks() :
+    _magazine(NULL) {
+  for (size_t i = 0; i < XMarkStripesMax; i++) {
+    _stacks[i] = NULL;
+  }
+}
+
+bool XMarkThreadLocalStacks::is_empty(const XMarkStripeSet* stripes) const {
+  for (size_t i = 0; i < stripes->nstripes(); i++) {
+    XMarkStack* const stack = _stacks[i];
+    if (stack != NULL) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+XMarkStack* XMarkThreadLocalStacks::allocate_stack(XMarkStackAllocator* allocator) {
+  if (_magazine == NULL) {
+    // Allocate new magazine
+    _magazine = allocator->alloc_magazine();
+    if (_magazine == NULL) {
+      return NULL;
+    }
+  }
+
+  XMarkStack* stack = NULL;
+
+  if (!_magazine->pop(stack)) {
+    // Magazine is empty, convert magazine into a new stack
+    _magazine->~XMarkStackMagazine();
+    stack = new ((void*)_magazine) XMarkStack();
+    _magazine = NULL;
+  }
+
+  return stack;
+}
+
+void XMarkThreadLocalStacks::free_stack(XMarkStackAllocator* allocator, XMarkStack* stack) {
+  for (;;) {
+    if (_magazine == NULL) {
+      // Convert stack into a new magazine
+      stack->~XMarkStack();
+      _magazine = new ((void*)stack) XMarkStackMagazine();
+      return;
+    }
+
+    if (_magazine->push(stack)) {
+      // Success
+      return;
+    }
+
+    // Free and uninstall full magazine
+    allocator->free_magazine(_magazine);
+    _magazine = NULL;
+  }
+}
+
+bool XMarkThreadLocalStacks::push_slow(XMarkStackAllocator* allocator,
+                                       XMarkStripe* stripe,
+                                       XMarkStack** stackp,
+                                       XMarkStackEntry entry,
+                                       bool publish) {
+  XMarkStack* stack = *stackp;
+
+  for (;;) {
+    if (stack == NULL) {
+      // Allocate and install new stack
+      *stackp = stack = allocate_stack(allocator);
+      if (stack == NULL) {
+        // Out of mark stack memory
+        return false;
+      }
+    }
+
+    if (stack->push(entry)) {
+      // Success
+      return true;
+    }
+
+    // Publish/Overflow and uninstall stack
+    stripe->publish_stack(stack, publish);
+    *stackp = stack = NULL;
+  }
+}
+
+bool XMarkThreadLocalStacks::pop_slow(XMarkStackAllocator* allocator,
+                                      XMarkStripe* stripe,
+                                      XMarkStack** stackp,
+                                      XMarkStackEntry& entry) {
+  XMarkStack* stack = *stackp;
+
+  for (;;) {
+    if (stack == NULL) {
+      // Try steal and install stack
+      *stackp = stack = stripe->steal_stack();
+      if (stack == NULL) {
+        // Nothing to steal
+        return false;
+      }
+    }
+
+    if (stack->pop(entry)) {
+      // Success
+      return true;
+    }
+
+    // Free and uninstall stack
+    free_stack(allocator, stack);
+    *stackp = stack = NULL;
+  }
+}
+
+bool XMarkThreadLocalStacks::flush(XMarkStackAllocator* allocator, XMarkStripeSet* stripes) {
+  bool flushed = false;
+
+  // Flush all stacks
+  for (size_t i = 0; i < stripes->nstripes(); i++) {
+    XMarkStripe* const stripe = stripes->stripe_at(i);
+    XMarkStack** const stackp = &_stacks[i];
+    XMarkStack* const stack = *stackp;
+    if (stack == NULL) {
+      continue;
+    }
+
+    // Free/Publish and uninstall stack
+    if (stack->is_empty()) {
+      free_stack(allocator, stack);
+    } else {
+      stripe->publish_stack(stack);
+      flushed = true;
+    }
+    *stackp = NULL;
+  }
+
+  return flushed;
+}
+
+void XMarkThreadLocalStacks::free(XMarkStackAllocator* allocator) {
+  // Free and uninstall magazine
+  if (_magazine != NULL) {
+    allocator->free_magazine(_magazine);
+    _magazine = NULL;
+  }
+}
diff --git a/src/hotspot/share/gc/x/xMarkStack.hpp b/src/hotspot/share/gc/x/xMarkStack.hpp
new file mode 100644
index 00000000000..e012b89749d
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMarkStack.hpp
@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XMARKSTACK_HPP
+#define SHARE_GC_X_XMARKSTACK_HPP
+
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xMarkStackEntry.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+template <typename T, size_t S>
+class XStack {
+private:
+  size_t        _top;
+  XStack<T, S>* _next;
+  T             _slots[S];
+
+  bool is_full() const;
+
+public:
+  XStack();
+
+  bool is_empty() const;
+
+  bool push(T value);
+  bool pop(T& value);
+
+  XStack<T, S>* next() const;
+  XStack<T, S>** next_addr();
+};
+
+template <typename T>
+class XStackList {
+private:
+  T* volatile _head;
+
+  T* encode_versioned_pointer(const T* stack, uint32_t version) const;
+  void decode_versioned_pointer(const T* vstack, T** stack, uint32_t* version) const;
+
+public:
+  XStackList();
+
+  bool is_empty() const;
+
+  void push(T* stack);
+  T* pop();
+
+  void clear();
+};
+
+using XMarkStack = XStack<XMarkStackEntry, XMarkStackSlots>;
+using XMarkStackList = XStackList<XMarkStack>;
+using XMarkStackMagazine = XStack<XMarkStack*, XMarkStackMagazineSlots>;
+using XMarkStackMagazineList = XStackList<XMarkStackMagazine>;
+
+static_assert(sizeof(XMarkStack) == XMarkStackSize, "XMarkStack size mismatch");
+static_assert(sizeof(XMarkStackMagazine) <= XMarkStackSize, "XMarkStackMagazine size too large");
+
+class XMarkStripe {
+private:
+  XCACHE_ALIGNED XMarkStackList _published;
+  XCACHE_ALIGNED XMarkStackList _overflowed;
+
+public:
+  XMarkStripe();
+
+  bool is_empty() const;
+
+  void publish_stack(XMarkStack* stack, bool publish = true);
+  XMarkStack* steal_stack();
+};
+
+class XMarkStripeSet {
+private:
+  size_t      _nstripes;
+  size_t      _nstripes_mask;
+  XMarkStripe _stripes[XMarkStripesMax];
+
+public:
+  XMarkStripeSet();
+
+  size_t nstripes() const;
+  void set_nstripes(size_t nstripes);
+
+  bool is_empty() const;
+
+  size_t stripe_id(const XMarkStripe* stripe) const;
+  XMarkStripe* stripe_at(size_t index);
+  XMarkStripe* stripe_next(XMarkStripe* stripe);
+  XMarkStripe* stripe_for_worker(uint nworkers, uint worker_id);
+  XMarkStripe* stripe_for_addr(uintptr_t addr);
+};
+
+class XMarkStackAllocator;
+
+class XMarkThreadLocalStacks {
+private:
+  XMarkStackMagazine* _magazine;
+  XMarkStack*         _stacks[XMarkStripesMax];
+
+  XMarkStack* allocate_stack(XMarkStackAllocator* allocator);
+  void free_stack(XMarkStackAllocator* allocator, XMarkStack* stack);
+
+  bool push_slow(XMarkStackAllocator* allocator,
+                 XMarkStripe* stripe,
+                 XMarkStack** stackp,
+                 XMarkStackEntry entry,
+                 bool publish);
+
+  bool pop_slow(XMarkStackAllocator* allocator,
+                XMarkStripe* stripe,
+                XMarkStack** stackp,
+                XMarkStackEntry& entry);
+
+public:
+  XMarkThreadLocalStacks();
+
+  bool is_empty(const XMarkStripeSet* stripes) const;
+
+  void install(XMarkStripeSet* stripes,
+               XMarkStripe* stripe,
+               XMarkStack* stack);
+
+  XMarkStack* steal(XMarkStripeSet* stripes,
+                    XMarkStripe* stripe);
+
+  bool push(XMarkStackAllocator* allocator,
+            XMarkStripeSet* stripes,
+            XMarkStripe* stripe,
+            XMarkStackEntry entry,
+            bool publish);
+
+  bool pop(XMarkStackAllocator* allocator,
+           XMarkStripeSet* stripes,
+           XMarkStripe* stripe,
+           XMarkStackEntry& entry);
+
+  bool flush(XMarkStackAllocator* allocator,
+             XMarkStripeSet* stripes);
+
+  void free(XMarkStackAllocator* allocator);
+};
+
+#endif // SHARE_GC_X_XMARKSTACK_HPP
diff --git a/src/hotspot/share/gc/x/xMarkStack.inline.hpp b/src/hotspot/share/gc/x/xMarkStack.inline.hpp
new file mode 100644
index 00000000000..95047c0954a
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMarkStack.inline.hpp
@@ -0,0 +1,266 @@
+/*
+ * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XMARKSTACK_INLINE_HPP
+#define SHARE_GC_X_XMARKSTACK_INLINE_HPP
+
+#include "gc/x/xMarkStack.hpp"
+
+#include "utilities/debug.hpp"
+#include "runtime/atomic.hpp"
+
+template <typename T, size_t S>
+inline XStack<T, S>::XStack() :
+    _top(0),
+    _next(NULL) {}
+
+template <typename T, size_t S>
+inline bool XStack<T, S>::is_empty() const {
+  return _top == 0;
+}
+
+template <typename T, size_t S>
+inline bool XStack<T, S>::is_full() const {
+  return _top == S;
+}
+
+template <typename T, size_t S>
+inline bool XStack<T, S>::push(T value) {
+  if (is_full()) {
+    return false;
+  }
+
+  _slots[_top++] = value;
+  return true;
+}
+
+template <typename T, size_t S>
+inline bool XStack<T, S>::pop(T& value) {
+  if (is_empty()) {
+    return false;
+  }
+
+  value = _slots[--_top];
+  return true;
+}
+
+template <typename T, size_t S>
+inline XStack<T, S>* XStack<T, S>::next() const {
+  return _next;
+}
+
+template <typename T, size_t S>
+inline XStack<T, S>** XStack<T, S>::next_addr() {
+  return &_next;
+}
+
+template <typename T>
+inline XStackList<T>::XStackList() :
+    _head(encode_versioned_pointer(NULL, 0)) {}
+
+template <typename T>
+inline T* XStackList<T>::encode_versioned_pointer(const T* stack, uint32_t version) const {
+  uint64_t addr;
+
+  if (stack == NULL) {
+    addr = (uint32_t)-1;
+  } else {
+    addr = ((uint64_t)stack - XMarkStackSpaceStart) >> XMarkStackSizeShift;
+  }
+
+  return (T*)((addr << 32) | (uint64_t)version);
+}
+
+template <typename T>
+inline void XStackList<T>::decode_versioned_pointer(const T* vstack, T** stack, uint32_t* version) const {
+  const uint64_t addr = (uint64_t)vstack >> 32;
+
+  if (addr == (uint32_t)-1) {
+    *stack = NULL;
+  } else {
+    *stack = (T*)((addr << XMarkStackSizeShift) + XMarkStackSpaceStart);
+  }
+
+  *version = (uint32_t)(uint64_t)vstack;
+}
+
+template <typename T>
+inline bool XStackList<T>::is_empty() const {
+  const T* vstack = _head;
+  T* stack = NULL;
+  uint32_t version = 0;
+
+  decode_versioned_pointer(vstack, &stack, &version);
+  return stack == NULL;
+}
+
+template <typename T>
+inline void XStackList<T>::push(T* stack) {
+  T* vstack = _head;
+  uint32_t version = 0;
+
+  for (;;) {
+    decode_versioned_pointer(vstack, stack->next_addr(), &version);
+    T* const new_vstack = encode_versioned_pointer(stack, version + 1);
+    T* const prev_vstack = Atomic::cmpxchg(&_head, vstack, new_vstack);
+    if (prev_vstack == vstack) {
+      // Success
+      break;
+    }
+
+    // Retry
+    vstack = prev_vstack;
+  }
+}
+
+template <typename T>
+inline T* XStackList<T>::pop() {
+  T* vstack = _head;
+  T* stack = NULL;
+  uint32_t version = 0;
+
+  for (;;) {
+    decode_versioned_pointer(vstack, &stack, &version);
+    if (stack == NULL) {
+      return NULL;
+    }
+
+    T* const new_vstack = encode_versioned_pointer(stack->next(), version + 1);
+    T* const prev_vstack = Atomic::cmpxchg(&_head, vstack, new_vstack);
+    if (prev_vstack == vstack) {
+      // Success
+      return stack;
+    }
+
+    // Retry
+    vstack = prev_vstack;
+  }
+}
+
+template <typename T>
+inline void XStackList<T>::clear() {
+  _head = encode_versioned_pointer(NULL, 0);
+}
+
+inline bool XMarkStripe::is_empty() const {
+  return _published.is_empty() && _overflowed.is_empty();
+}
+
+inline void XMarkStripe::publish_stack(XMarkStack* stack, bool publish) {
+  // A stack is published either on the published list or the overflowed
+  // list. The published list is used by mutators publishing stacks for GC
+  // workers to work on, while the overflowed list is used by GC workers
+  // to publish stacks that overflowed. The intention here is to avoid
+  // contention between mutators and GC workers as much as possible, while
+  // still allowing GC workers to help out and steal work from each other.
+  if (publish) {
+    _published.push(stack);
+  } else {
+    _overflowed.push(stack);
+  }
+}
+
+inline XMarkStack* XMarkStripe::steal_stack() {
+  // Steal overflowed stacks first, then published stacks
+  XMarkStack* const stack = _overflowed.pop();
+  if (stack != NULL) {
+    return stack;
+  }
+
+  return _published.pop();
+}
+
+inline size_t XMarkStripeSet::nstripes() const {
+  return _nstripes;
+}
+
+inline size_t XMarkStripeSet::stripe_id(const XMarkStripe* stripe) const {
+  const size_t index = ((uintptr_t)stripe - (uintptr_t)_stripes) / sizeof(XMarkStripe);
+  assert(index < _nstripes, "Invalid index");
+  return index;
+}
+
+inline XMarkStripe* XMarkStripeSet::stripe_at(size_t index) {
+  assert(index < _nstripes, "Invalid index");
+  return &_stripes[index];
+}
+
+inline XMarkStripe* XMarkStripeSet::stripe_next(XMarkStripe* stripe) {
+  const size_t index = (stripe_id(stripe) + 1) & _nstripes_mask;
+  assert(index < _nstripes, "Invalid index");
+  return &_stripes[index];
+}
+
+inline XMarkStripe* XMarkStripeSet::stripe_for_addr(uintptr_t addr) {
+  const size_t index = (addr >> XMarkStripeShift) & _nstripes_mask;
+  assert(index < _nstripes, "Invalid index");
+  return &_stripes[index];
+}
+
+inline void XMarkThreadLocalStacks::install(XMarkStripeSet* stripes,
+                                            XMarkStripe* stripe,
+                                            XMarkStack* stack) {
+  XMarkStack** const stackp = &_stacks[stripes->stripe_id(stripe)];
+  assert(*stackp == NULL, "Should be empty");
+  *stackp = stack;
+}
+
+inline XMarkStack* XMarkThreadLocalStacks::steal(XMarkStripeSet* stripes,
+                                                 XMarkStripe* stripe) {
+  XMarkStack** const stackp = &_stacks[stripes->stripe_id(stripe)];
+  XMarkStack* const stack = *stackp;
+  if (stack != NULL) {
+    *stackp = NULL;
+  }
+
+  return stack;
+}
+
+inline bool XMarkThreadLocalStacks::push(XMarkStackAllocator* allocator,
+                                         XMarkStripeSet* stripes,
+                                         XMarkStripe* stripe,
+                                         XMarkStackEntry entry,
+                                         bool publish) {
+  XMarkStack** const stackp = &_stacks[stripes->stripe_id(stripe)];
+  XMarkStack* const stack = *stackp;
+  if (stack != NULL && stack->push(entry)) {
+    return true;
+  }
+
+  return push_slow(allocator, stripe, stackp, entry, publish);
+}
+
+inline bool XMarkThreadLocalStacks::pop(XMarkStackAllocator* allocator,
+                                        XMarkStripeSet* stripes,
+                                        XMarkStripe* stripe,
+                                        XMarkStackEntry& entry) {
+  XMarkStack** const stackp = &_stacks[stripes->stripe_id(stripe)];
+  XMarkStack* const stack = *stackp;
+  if (stack != NULL && stack->pop(entry)) {
+    return true;
+  }
+
+  return pop_slow(allocator, stripe, stackp, entry);
+}
+
+#endif // SHARE_GC_X_XMARKSTACK_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xMarkStackAllocator.cpp b/src/hotspot/share/gc/x/xMarkStackAllocator.cpp
new file mode 100644
index 00000000000..5e7c1d223a5
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMarkStackAllocator.cpp
@@ -0,0 +1,221 @@
+/*
+ * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/xLock.inline.hpp"
+#include "gc/x/xMarkStack.inline.hpp"
+#include "gc/x/xMarkStackAllocator.hpp"
+#include "logging/log.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/os.hpp"
+#include "utilities/debug.hpp"
+
+uintptr_t XMarkStackSpaceStart;
+
+XMarkStackSpace::XMarkStackSpace() :
+    _expand_lock(),
+    _start(0),
+    _top(0),
+    _end(0) {
+  assert(ZMarkStackSpaceLimit >= XMarkStackSpaceExpandSize, "ZMarkStackSpaceLimit too small");
+
+  // Reserve address space
+  const size_t size = ZMarkStackSpaceLimit;
+  const uintptr_t addr = (uintptr_t)os::reserve_memory(size, !ExecMem, mtGC);
+  if (addr == 0) {
+    log_error_pd(gc, marking)("Failed to reserve address space for mark stacks");
+    return;
+  }
+
+  // Successfully initialized
+  _start = _top = _end = addr;
+
+  // Register mark stack space start
+  XMarkStackSpaceStart = _start;
+
+  // Prime space
+  _end += expand_space();
+}
+
+bool XMarkStackSpace::is_initialized() const {
+  return _start != 0;
+}
+
+size_t XMarkStackSpace::size() const {
+  return _end - _start;
+}
+
+size_t XMarkStackSpace::used() const {
+  return _top - _start;
+}
+
+size_t XMarkStackSpace::expand_space() {
+  const size_t expand_size = XMarkStackSpaceExpandSize;
+  const size_t old_size = size();
+  const size_t new_size = old_size + expand_size;
+
+  if (new_size > ZMarkStackSpaceLimit) {
+    // Expansion limit reached. This is a fatal error since we
+    // currently can't recover from running out of mark stack space.
+    fatal("Mark stack space exhausted. Use -XX:ZMarkStackSpaceLimit=<size> to increase the "
+          "maximum number of bytes allocated for mark stacks. Current limit is " SIZE_FORMAT "M.",
+          ZMarkStackSpaceLimit / M);
+  }
+
+  log_debug(gc, marking)("Expanding mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M",
+                         old_size / M, new_size / M);
+
+  // Expand
+  os::commit_memory_or_exit((char*)_end, expand_size, false /* executable */, "Mark stack space");
+
+  return expand_size;
+}
+
+size_t XMarkStackSpace::shrink_space() {
+  // Shrink to what is currently used
+  const size_t old_size = size();
+  const size_t new_size = align_up(used(), XMarkStackSpaceExpandSize);
+  const size_t shrink_size = old_size - new_size;
+
+  if (shrink_size > 0) {
+    // Shrink
+    log_debug(gc, marking)("Shrinking mark stack space: " SIZE_FORMAT "M->" SIZE_FORMAT "M",
+                           old_size / M, new_size / M);
+
+    const uintptr_t shrink_start = _end - shrink_size;
+    os::uncommit_memory((char*)shrink_start, shrink_size, false /* executable */);
+  }
+
+  return shrink_size;
+}
+
+uintptr_t XMarkStackSpace::alloc_space(size_t size) {
+  uintptr_t top = Atomic::load(&_top);
+
+  for (;;) {
+    const uintptr_t end = Atomic::load(&_end);
+    const uintptr_t new_top = top + size;
+    if (new_top > end) {
+      // Not enough space left
+      return 0;
+    }
+
+    const uintptr_t prev_top = Atomic::cmpxchg(&_top, top, new_top);
+    if (prev_top == top) {
+      // Success
+      return top;
+    }
+
+    // Retry
+    top = prev_top;
+  }
+}
+
+uintptr_t XMarkStackSpace::expand_and_alloc_space(size_t size) {
+  XLocker<XLock> locker(&_expand_lock);
+
+  // Retry allocation before expanding
+  uintptr_t addr = alloc_space(size);
+  if (addr != 0) {
+    return addr;
+  }
+
+  // Expand
+  const size_t expand_size = expand_space();
+
+  // Increment top before end to make sure another
+  // thread can't steal out newly expanded space.
+  addr = Atomic::fetch_and_add(&_top, size);
+  Atomic::add(&_end, expand_size);
+
+  return addr;
+}
+
+uintptr_t XMarkStackSpace::alloc(size_t size) {
+  assert(size <= XMarkStackSpaceExpandSize, "Invalid size");
+
+  const uintptr_t addr = alloc_space(size);
+  if (addr != 0) {
+    return addr;
+  }
+
+  return expand_and_alloc_space(size);
+}
+
+void XMarkStackSpace::free() {
+  _end -= shrink_space();
+  _top = _start;
+}
+
+XMarkStackAllocator::XMarkStackAllocator() :
+    _freelist(),
+    _space() {}
+
+bool XMarkStackAllocator::is_initialized() const {
+  return _space.is_initialized();
+}
+
+size_t XMarkStackAllocator::size() const {
+  return _space.size();
+}
+
+XMarkStackMagazine* XMarkStackAllocator::create_magazine_from_space(uintptr_t addr, size_t size) {
+  assert(is_aligned(size, XMarkStackSize), "Invalid size");
+
+  // Use first stack as magazine
+  XMarkStackMagazine* const magazine = new ((void*)addr) XMarkStackMagazine();
+  for (size_t i = XMarkStackSize; i < size; i += XMarkStackSize) {
+    XMarkStack* const stack = new ((void*)(addr + i)) XMarkStack();
+    const bool success = magazine->push(stack);
+    assert(success, "Magazine should never get full");
+  }
+
+  return magazine;
+}
+
+XMarkStackMagazine* XMarkStackAllocator::alloc_magazine() {
+  // Try allocating from the free list first
+  XMarkStackMagazine* const magazine = _freelist.pop();
+  if (magazine != NULL) {
+    return magazine;
+  }
+
+  // Allocate new magazine
+  const uintptr_t addr = _space.alloc(XMarkStackMagazineSize);
+  if (addr == 0) {
+    return NULL;
+  }
+
+  return create_magazine_from_space(addr, XMarkStackMagazineSize);
+}
+
+void XMarkStackAllocator::free_magazine(XMarkStackMagazine* magazine) {
+  _freelist.push(magazine);
+}
+
+void XMarkStackAllocator::free() {
+  _freelist.clear();
+  _space.free();
+}
diff --git a/src/hotspot/share/gc/x/xMarkStackAllocator.hpp b/src/hotspot/share/gc/x/xMarkStackAllocator.hpp
new file mode 100644
index 00000000000..5e81ae284cf
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMarkStackAllocator.hpp
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XMARKSTACKALLOCATOR_HPP
+#define SHARE_GC_X_XMARKSTACKALLOCATOR_HPP
+
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xLock.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class XMarkStackSpace {
+private:
+  XLock              _expand_lock;
+  uintptr_t          _start;
+  volatile uintptr_t _top;
+  volatile uintptr_t _end;
+
+  size_t used() const;
+
+  size_t expand_space();
+  size_t shrink_space();
+
+  uintptr_t alloc_space(size_t size);
+  uintptr_t expand_and_alloc_space(size_t size);
+
+public:
+  XMarkStackSpace();
+
+  bool is_initialized() const;
+
+  size_t size() const;
+
+  uintptr_t alloc(size_t size);
+  void free();
+};
+
+class XMarkStackAllocator {
+private:
+  XCACHE_ALIGNED XMarkStackMagazineList _freelist;
+  XCACHE_ALIGNED XMarkStackSpace        _space;
+
+  XMarkStackMagazine* create_magazine_from_space(uintptr_t addr, size_t size);
+
+public:
+  XMarkStackAllocator();
+
+  bool is_initialized() const;
+
+  size_t size() const;
+
+  XMarkStackMagazine* alloc_magazine();
+  void free_magazine(XMarkStackMagazine* magazine);
+
+  void free();
+};
+
+#endif // SHARE_GC_X_XMARKSTACKALLOCATOR_HPP
diff --git a/src/hotspot/share/gc/x/xMarkStackEntry.hpp b/src/hotspot/share/gc/x/xMarkStackEntry.hpp
new file mode 100644
index 00000000000..61df1798df2
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMarkStackEntry.hpp
@@ -0,0 +1,142 @@
+/*
+ * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XMARKSTACKENTRY_HPP
+#define SHARE_GC_X_XMARKSTACKENTRY_HPP
+
+#include "gc/x/xBitField.hpp"
+#include "memory/allocation.hpp"
+
+//
+// Mark stack entry layout
+// -----------------------
+//
+//  Object entry
+//  ------------
+//
+//   6
+//   3                                                                5 4 3 2 1 0
+//  +------------------------------------------------------------------+-+-+-+-+-+
+//  |11111111 11111111 11111111 11111111 11111111 11111111 11111111 111|1|1|1|1|1|
+//  +------------------------------------------------------------------+-+-+-+-+-+
+//  |                                                                  | | | | |
+//  |                                            4-4 Mark Flag (1-bit) * | | | |
+//  |                                                                    | | | |
+//  |                                    3-3 Increment Live Flag (1-bit) * | | |
+//  |                                                                      | | |
+//  |                                              2-2 Follow Flag (1-bit) * | |
+//  |                                                                        | |
+//  |                                         1-1 Partial Array Flag (1-bit) * |
+//  |                                                                          |
+//  |                                                   0-0 Final Flag (1-bit) *
+//  |
+//  * 63-5 Object Address (59-bits)
+//
+//
+//  Partial array entry
+//  -------------------
+//
+//   6                                 3  3
+//   3                                 2  1                               2 1 0
+//  +------------------------------------+---------------------------------+-+-+
+//  |11111111 11111111 11111111 11111111 |11111111 11111111 11111111 111111|1|1|
+//  +------------------------------------+---------------------------------+-+-+
+//  |                                    |                                 | |
+//  |                                    |  1-1 Partial Array Flag (1-bit) * |
+//  |                                    |                                   |
+//  |                                    |            0-0 Final Flag (1-bit) *
+//  |                                    |
+//  |                                    * 31-2 Partial Array Length (30-bits)
+//  |
+//  * 63-32 Partial Array Address Offset (32-bits)
+//
+
+class XMarkStackEntry  {
+private:
+  typedef XBitField<uint64_t, bool,      0,  1>  field_finalizable;
+  typedef XBitField<uint64_t, bool,      1,  1>  field_partial_array;
+  typedef XBitField<uint64_t, bool,      2,  1>  field_follow;
+  typedef XBitField<uint64_t, bool,      3,  1>  field_inc_live;
+  typedef XBitField<uint64_t, bool,      4,  1>  field_mark;
+  typedef XBitField<uint64_t, uintptr_t, 5,  59> field_object_address;
+  typedef XBitField<uint64_t, size_t,    2,  30> field_partial_array_length;
+  typedef XBitField<uint64_t, size_t,    32, 32> field_partial_array_offset;
+
+  uint64_t _entry;
+
+public:
+  XMarkStackEntry() {
+    // This constructor is intentionally left empty and does not initialize
+    // _entry to allow it to be optimized out when instantiating XMarkStack,
+    // which has a long array of XMarkStackEntry elements, but doesn't care
+    // what _entry is initialized to.
+  }
+
+  XMarkStackEntry(uintptr_t object_address, bool mark, bool inc_live, bool follow, bool finalizable) :
+      _entry(field_object_address::encode(object_address) |
+             field_mark::encode(mark) |
+             field_inc_live::encode(inc_live) |
+             field_follow::encode(follow) |
+             field_partial_array::encode(false) |
+             field_finalizable::encode(finalizable)) {}
+
+  XMarkStackEntry(size_t partial_array_offset, size_t partial_array_length, bool finalizable) :
+      _entry(field_partial_array_offset::encode(partial_array_offset) |
+             field_partial_array_length::encode(partial_array_length) |
+             field_partial_array::encode(true) |
+             field_finalizable::encode(finalizable)) {}
+
+  bool finalizable() const {
+    return field_finalizable::decode(_entry);
+  }
+
+  bool partial_array() const {
+    return field_partial_array::decode(_entry);
+  }
+
+  size_t partial_array_offset() const {
+    return field_partial_array_offset::decode(_entry);
+  }
+
+  size_t partial_array_length() const {
+    return field_partial_array_length::decode(_entry);
+  }
+
+  bool follow() const {
+    return field_follow::decode(_entry);
+  }
+
+  bool inc_live() const {
+    return field_inc_live::decode(_entry);
+  }
+
+  bool mark() const {
+    return field_mark::decode(_entry);
+  }
+
+  uintptr_t object_address() const {
+    return field_object_address::decode(_entry);
+  }
+};
+
+#endif // SHARE_GC_X_XMARKSTACKENTRY_HPP
diff --git a/src/hotspot/share/gc/x/xMarkTerminate.hpp b/src/hotspot/share/gc/x/xMarkTerminate.hpp
new file mode 100644
index 00000000000..28f18f6e1cb
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMarkTerminate.hpp
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XMARKTERMINATE_HPP
+#define SHARE_GC_X_XMARKTERMINATE_HPP
+
+#include "gc/x/xGlobals.hpp"
+#include "memory/allocation.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class XMarkTerminate {
+private:
+  uint                         _nworkers;
+  XCACHE_ALIGNED volatile uint _nworking_stage0;
+  volatile uint                _nworking_stage1;
+
+  bool enter_stage(volatile uint* nworking_stage);
+  void exit_stage(volatile uint* nworking_stage);
+  bool try_exit_stage(volatile uint* nworking_stage);
+
+public:
+  XMarkTerminate();
+
+  void reset(uint nworkers);
+
+  bool enter_stage0();
+  void exit_stage0();
+  bool try_exit_stage0();
+
+  bool enter_stage1();
+  bool try_exit_stage1();
+};
+
+#endif // SHARE_GC_X_XMARKTERMINATE_HPP
diff --git a/src/hotspot/share/gc/x/xMarkTerminate.inline.hpp b/src/hotspot/share/gc/x/xMarkTerminate.inline.hpp
new file mode 100644
index 00000000000..e4b9256ba6b
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMarkTerminate.inline.hpp
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XMARKTERMINATE_INLINE_HPP
+#define SHARE_GC_X_XMARKTERMINATE_INLINE_HPP
+
+#include "gc/x/xMarkTerminate.hpp"
+
+#include "runtime/atomic.hpp"
+
+inline XMarkTerminate::XMarkTerminate() :
+    _nworkers(0),
+    _nworking_stage0(0),
+    _nworking_stage1(0) {}
+
+inline bool XMarkTerminate::enter_stage(volatile uint* nworking_stage) {
+  return Atomic::sub(nworking_stage, 1u) == 0;
+}
+
+inline void XMarkTerminate::exit_stage(volatile uint* nworking_stage) {
+  Atomic::add(nworking_stage, 1u);
+}
+
+inline bool XMarkTerminate::try_exit_stage(volatile uint* nworking_stage) {
+  uint nworking = Atomic::load(nworking_stage);
+
+  for (;;) {
+    if (nworking == 0) {
+      return false;
+    }
+
+    const uint new_nworking = nworking + 1;
+    const uint prev_nworking = Atomic::cmpxchg(nworking_stage, nworking, new_nworking);
+    if (prev_nworking == nworking) {
+      // Success
+      return true;
+    }
+
+    // Retry
+    nworking = prev_nworking;
+  }
+}
+
+inline void XMarkTerminate::reset(uint nworkers) {
+  _nworkers = _nworking_stage0 = _nworking_stage1 = nworkers;
+}
+
+inline bool XMarkTerminate::enter_stage0() {
+  return enter_stage(&_nworking_stage0);
+}
+
+inline void XMarkTerminate::exit_stage0() {
+  exit_stage(&_nworking_stage0);
+}
+
+inline bool XMarkTerminate::try_exit_stage0() {
+  return try_exit_stage(&_nworking_stage0);
+}
+
+inline bool XMarkTerminate::enter_stage1() {
+  return enter_stage(&_nworking_stage1);
+}
+
+inline bool XMarkTerminate::try_exit_stage1() {
+  return try_exit_stage(&_nworking_stage1);
+}
+
+#endif // SHARE_GC_X_XMARKTERMINATE_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xMemory.cpp b/src/hotspot/share/gc/x/xMemory.cpp
new file mode 100644
index 00000000000..bdfa45339c2
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMemory.cpp
@@ -0,0 +1,220 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xList.inline.hpp"
+#include "gc/x/xLock.inline.hpp"
+#include "gc/x/xMemory.inline.hpp"
+
+XMemory* XMemoryManager::create(uintptr_t start, size_t size) {
+  XMemory* const area = new XMemory(start, size);
+  if (_callbacks._create != NULL) {
+    _callbacks._create(area);
+  }
+  return area;
+}
+
+void XMemoryManager::destroy(XMemory* area) {
+  if (_callbacks._destroy != NULL) {
+    _callbacks._destroy(area);
+  }
+  delete area;
+}
+
+void XMemoryManager::shrink_from_front(XMemory* area, size_t size) {
+  if (_callbacks._shrink_from_front != NULL) {
+    _callbacks._shrink_from_front(area, size);
+  }
+  area->shrink_from_front(size);
+}
+
+void XMemoryManager::shrink_from_back(XMemory* area, size_t size) {
+  if (_callbacks._shrink_from_back != NULL) {
+    _callbacks._shrink_from_back(area, size);
+  }
+  area->shrink_from_back(size);
+}
+
+void XMemoryManager::grow_from_front(XMemory* area, size_t size) {
+  if (_callbacks._grow_from_front != NULL) {
+    _callbacks._grow_from_front(area, size);
+  }
+  area->grow_from_front(size);
+}
+
+void XMemoryManager::grow_from_back(XMemory* area, size_t size) {
+  if (_callbacks._grow_from_back != NULL) {
+    _callbacks._grow_from_back(area, size);
+  }
+  area->grow_from_back(size);
+}
+
+XMemoryManager::Callbacks::Callbacks() :
+    _create(NULL),
+    _destroy(NULL),
+    _shrink_from_front(NULL),
+    _shrink_from_back(NULL),
+    _grow_from_front(NULL),
+    _grow_from_back(NULL) {}
+
+XMemoryManager::XMemoryManager() :
+    _freelist(),
+    _callbacks() {}
+
+void XMemoryManager::register_callbacks(const Callbacks& callbacks) {
+  _callbacks = callbacks;
+}
+
+uintptr_t XMemoryManager::peek_low_address() const {
+  XLocker<XLock> locker(&_lock);
+
+  const XMemory* const area = _freelist.first();
+  if (area != NULL) {
+    return area->start();
+  }
+
+  // Out of memory
+  return UINTPTR_MAX;
+}
+
+uintptr_t XMemoryManager::alloc_low_address(size_t size) {
+  XLocker<XLock> locker(&_lock);
+
+  XListIterator<XMemory> iter(&_freelist);
+  for (XMemory* area; iter.next(&area);) {
+    if (area->size() >= size) {
+      if (area->size() == size) {
+        // Exact match, remove area
+        const uintptr_t start = area->start();
+        _freelist.remove(area);
+        destroy(area);
+        return start;
+      } else {
+        // Larger than requested, shrink area
+        const uintptr_t start = area->start();
+        shrink_from_front(area, size);
+        return start;
+      }
+    }
+  }
+
+  // Out of memory
+  return UINTPTR_MAX;
+}
+
+uintptr_t XMemoryManager::alloc_low_address_at_most(size_t size, size_t* allocated) {
+  XLocker<XLock> locker(&_lock);
+
+  XMemory* area = _freelist.first();
+  if (area != NULL) {
+    if (area->size() <= size) {
+      // Smaller than or equal to requested, remove area
+      const uintptr_t start = area->start();
+      *allocated = area->size();
+      _freelist.remove(area);
+      destroy(area);
+      return start;
+    } else {
+      // Larger than requested, shrink area
+      const uintptr_t start = area->start();
+      shrink_from_front(area, size);
+      *allocated = size;
+      return start;
+    }
+  }
+
+  // Out of memory
+  *allocated = 0;
+  return UINTPTR_MAX;
+}
+
+uintptr_t XMemoryManager::alloc_high_address(size_t size) {
+  XLocker<XLock> locker(&_lock);
+
+  XListReverseIterator<XMemory> iter(&_freelist);
+  for (XMemory* area; iter.next(&area);) {
+    if (area->size() >= size) {
+      if (area->size() == size) {
+        // Exact match, remove area
+        const uintptr_t start = area->start();
+        _freelist.remove(area);
+        destroy(area);
+        return start;
+      } else {
+        // Larger than requested, shrink area
+        shrink_from_back(area, size);
+        return area->end();
+      }
+    }
+  }
+
+  // Out of memory
+  return UINTPTR_MAX;
+}
+
+void XMemoryManager::free(uintptr_t start, size_t size) {
+  assert(start != UINTPTR_MAX, "Invalid address");
+  const uintptr_t end = start + size;
+
+  XLocker<XLock> locker(&_lock);
+
+  XListIterator<XMemory> iter(&_freelist);
+  for (XMemory* area; iter.next(&area);) {
+    if (start < area->start()) {
+      XMemory* const prev = _freelist.prev(area);
+      if (prev != NULL && start == prev->end()) {
+        if (end == area->start()) {
+          // Merge with prev and current area
+          grow_from_back(prev, size + area->size());
+          _freelist.remove(area);
+          delete area;
+        } else {
+          // Merge with prev area
+          grow_from_back(prev, size);
+        }
+      } else if (end == area->start()) {
+        // Merge with current area
+        grow_from_front(area, size);
+      } else {
+        // Insert new area before current area
+        assert(end < area->start(), "Areas must not overlap");
+        XMemory* const new_area = create(start, size);
+        _freelist.insert_before(area, new_area);
+      }
+
+      // Done
+      return;
+    }
+  }
+
+  // Insert last
+  XMemory* const last = _freelist.last();
+  if (last != NULL && start == last->end()) {
+    // Merge with last area
+    grow_from_back(last, size);
+  } else {
+    // Insert new area last
+    XMemory* const new_area = create(start, size);
+    _freelist.insert_last(new_area);
+  }
+}
diff --git a/src/hotspot/share/gc/x/xMemory.hpp b/src/hotspot/share/gc/x/xMemory.hpp
new file mode 100644
index 00000000000..2c3739cb44a
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMemory.hpp
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XMEMORY_HPP
+#define SHARE_GC_X_XMEMORY_HPP
+
+#include "gc/x/xList.hpp"
+#include "gc/x/xLock.hpp"
+#include "memory/allocation.hpp"
+
+class XMemory : public CHeapObj<mtGC> {
+  friend class XList<XMemory>;
+
+private:
+  uintptr_t          _start;
+  uintptr_t          _end;
+  XListNode<XMemory> _node;
+
+public:
+  XMemory(uintptr_t start, size_t size);
+
+  uintptr_t start() const;
+  uintptr_t end() const;
+  size_t size() const;
+
+  void shrink_from_front(size_t size);
+  void shrink_from_back(size_t size);
+  void grow_from_front(size_t size);
+  void grow_from_back(size_t size);
+};
+
+class XMemoryManager {
+public:
+  typedef void (*CreateDestroyCallback)(const XMemory* area);
+  typedef void (*ResizeCallback)(const XMemory* area, size_t size);
+
+  struct Callbacks {
+    CreateDestroyCallback _create;
+    CreateDestroyCallback _destroy;
+    ResizeCallback        _shrink_from_front;
+    ResizeCallback        _shrink_from_back;
+    ResizeCallback        _grow_from_front;
+    ResizeCallback        _grow_from_back;
+
+    Callbacks();
+  };
+
+private:
+  mutable XLock  _lock;
+  XList<XMemory> _freelist;
+  Callbacks      _callbacks;
+
+  XMemory* create(uintptr_t start, size_t size);
+  void destroy(XMemory* area);
+  void shrink_from_front(XMemory* area, size_t size);
+  void shrink_from_back(XMemory* area, size_t size);
+  void grow_from_front(XMemory* area, size_t size);
+  void grow_from_back(XMemory* area, size_t size);
+
+public:
+  XMemoryManager();
+
+  void register_callbacks(const Callbacks& callbacks);
+
+  uintptr_t peek_low_address() const;
+  uintptr_t alloc_low_address(size_t size);
+  uintptr_t alloc_low_address_at_most(size_t size, size_t* allocated);
+  uintptr_t alloc_high_address(size_t size);
+
+  void free(uintptr_t start, size_t size);
+};
+
+#endif // SHARE_GC_X_XMEMORY_HPP
diff --git a/src/hotspot/share/gc/x/xMemory.inline.hpp b/src/hotspot/share/gc/x/xMemory.inline.hpp
new file mode 100644
index 00000000000..332cdae9160
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMemory.inline.hpp
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XMEMORY_INLINE_HPP
+#define SHARE_GC_X_XMEMORY_INLINE_HPP
+
+#include "gc/x/xMemory.hpp"
+
+#include "gc/x/xList.inline.hpp"
+#include "utilities/debug.hpp"
+
+inline XMemory::XMemory(uintptr_t start, size_t size) :
+    _start(start),
+    _end(start + size) {}
+
+inline uintptr_t XMemory::start() const {
+  return _start;
+}
+
+inline uintptr_t XMemory::end() const {
+  return _end;
+}
+
+inline size_t XMemory::size() const {
+  return end() - start();
+}
+
+inline void XMemory::shrink_from_front(size_t size) {
+  assert(this->size() > size, "Too small");
+  _start += size;
+}
+
+inline void XMemory::shrink_from_back(size_t size) {
+  assert(this->size() > size, "Too small");
+  _end -= size;
+}
+
+inline void XMemory::grow_from_front(size_t size) {
+  assert(start() >= size, "Too big");
+  _start -= size;
+}
+
+inline void XMemory::grow_from_back(size_t size) {
+  _end += size;
+}
+
+#endif // SHARE_GC_X_XMEMORY_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zMessagePort.hpp b/src/hotspot/share/gc/x/xMessagePort.hpp
similarity index 79%
rename from src/hotspot/share/gc/z/zMessagePort.hpp
rename to src/hotspot/share/gc/x/xMessagePort.hpp
index 364eddad9f2..20565253796 100644
--- a/src/hotspot/share/gc/z/zMessagePort.hpp
+++ b/src/hotspot/share/gc/x/xMessagePort.hpp
@@ -21,28 +21,28 @@
  * questions.
  */
 
-#ifndef SHARE_GC_Z_ZMESSAGEPORT_HPP
-#define SHARE_GC_Z_ZMESSAGEPORT_HPP
+#ifndef SHARE_GC_X_XMESSAGEPORT_HPP
+#define SHARE_GC_X_XMESSAGEPORT_HPP
 
-#include "gc/z/zFuture.hpp"
-#include "gc/z/zList.hpp"
+#include "gc/x/xFuture.hpp"
+#include "gc/x/xList.hpp"
 #include "runtime/mutex.hpp"
 
-template <typename T> class ZMessageRequest;
+template <typename T> class XMessageRequest;
 
 template <typename T>
-class ZMessagePort {
+class XMessagePort {
 private:
-  typedef ZMessageRequest<T> Request;
+  typedef XMessageRequest<T> Request;
 
   mutable Monitor _monitor;
   bool            _has_message;
   T               _message;
   uint64_t        _seqnum;
-  ZList<Request>  _queue;
+  XList<Request>  _queue;
 
 public:
-  ZMessagePort();
+  XMessagePort();
 
   bool is_busy() const;
 
@@ -53,9 +53,9 @@ public:
   void ack();
 };
 
-class ZRendezvousPort {
+class XRendezvousPort {
 private:
-  ZMessagePort<bool> _port;
+  XMessagePort<bool> _port;
 
 public:
   void signal();
@@ -63,4 +63,4 @@ public:
   void ack();
 };
 
-#endif // SHARE_GC_Z_ZMESSAGEPORT_HPP
+#endif // SHARE_GC_X_XMESSAGEPORT_HPP
diff --git a/src/hotspot/share/gc/z/zMessagePort.inline.hpp b/src/hotspot/share/gc/x/xMessagePort.inline.hpp
similarity index 82%
rename from src/hotspot/share/gc/z/zMessagePort.inline.hpp
rename to src/hotspot/share/gc/x/xMessagePort.inline.hpp
index e9e9ba539a2..8007a80eacd 100644
--- a/src/hotspot/share/gc/z/zMessagePort.inline.hpp
+++ b/src/hotspot/share/gc/x/xMessagePort.inline.hpp
@@ -21,24 +21,24 @@
  * questions.
  */
 
-#ifndef SHARE_GC_Z_ZMESSAGEPORT_INLINE_HPP
-#define SHARE_GC_Z_ZMESSAGEPORT_INLINE_HPP
+#ifndef SHARE_GC_X_XMESSAGEPORT_INLINE_HPP
+#define SHARE_GC_X_XMESSAGEPORT_INLINE_HPP
 
-#include "gc/z/zMessagePort.hpp"
+#include "gc/x/xMessagePort.hpp"
 
-#include "gc/z/zFuture.inline.hpp"
-#include "gc/z/zList.inline.hpp"
+#include "gc/x/xFuture.inline.hpp"
+#include "gc/x/xList.inline.hpp"
 #include "runtime/mutexLocker.hpp"
 
 template <typename T>
-class ZMessageRequest : public StackObj {
-  friend class ZList<ZMessageRequest>;
+class XMessageRequest : public StackObj {
+  friend class XList<XMessageRequest>;
 
 private:
   T                          _message;
   uint64_t                   _seqnum;
-  ZFuture<T>                 _result;
-  ZListNode<ZMessageRequest> _node;
+  XFuture<T>                 _result;
+  XListNode<XMessageRequest> _node;
 
 public:
   void initialize(T message, uint64_t seqnum) {
@@ -65,20 +65,20 @@ public:
 };
 
 template <typename T>
-inline ZMessagePort<T>::ZMessagePort() :
-    _monitor(Monitor::nosafepoint, "ZMessagePort_lock"),
+inline XMessagePort<T>::XMessagePort() :
+    _monitor(Monitor::nosafepoint, "XMessagePort_lock"),
     _has_message(false),
     _seqnum(0),
     _queue() {}
 
 template <typename T>
-inline bool ZMessagePort<T>::is_busy() const {
+inline bool XMessagePort<T>::is_busy() const {
   MonitorLocker ml(&_monitor, Monitor::_no_safepoint_check_flag);
   return _has_message;
 }
 
 template <typename T>
-inline void ZMessagePort<T>::send_sync(const T& message) {
+inline void XMessagePort<T>::send_sync(const T& message) {
   Request request;
 
   {
@@ -105,7 +105,7 @@ inline void ZMessagePort<T>::send_sync(const T& message) {
 }
 
 template <typename T>
-inline void ZMessagePort<T>::send_async(const T& message) {
+inline void XMessagePort<T>::send_async(const T& message) {
   MonitorLocker ml(&_monitor, Monitor::_no_safepoint_check_flag);
   if (!_has_message) {
     // Post message
@@ -116,7 +116,7 @@ inline void ZMessagePort<T>::send_async(const T& message) {
 }
 
 template <typename T>
-inline T ZMessagePort<T>::receive() {
+inline T XMessagePort<T>::receive() {
   MonitorLocker ml(&_monitor, Monitor::_no_safepoint_check_flag);
 
   // Wait for message
@@ -137,7 +137,7 @@ inline T ZMessagePort<T>::receive() {
 }
 
 template <typename T>
-inline void ZMessagePort<T>::ack() {
+inline void XMessagePort<T>::ack() {
   MonitorLocker ml(&_monitor, Monitor::_no_safepoint_check_flag);
 
   if (!_has_message) {
@@ -146,7 +146,7 @@ inline void ZMessagePort<T>::ack() {
   }
 
   // Satisfy requests (and duplicates) in queue
-  ZListIterator<Request> iter(&_queue);
+  XListIterator<Request> iter(&_queue);
   for (Request* request; iter.next(&request);) {
     if (request->message() == _message && request->seqnum() < _seqnum) {
       // Dequeue and satisfy request. Note that the dequeue operation must
@@ -166,16 +166,16 @@ inline void ZMessagePort<T>::ack() {
   }
 }
 
-inline void ZRendezvousPort::signal() {
+inline void XRendezvousPort::signal() {
   _port.send_sync(true /* ignored */);
 }
 
-inline void ZRendezvousPort::wait() {
+inline void XRendezvousPort::wait() {
   _port.receive();
 }
 
-inline void ZRendezvousPort::ack() {
+inline void XRendezvousPort::ack() {
   _port.ack();
 }
 
-#endif // SHARE_GC_Z_ZMESSAGEPORT_INLINE_HPP
+#endif // SHARE_GC_X_XMESSAGEPORT_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xMetronome.cpp b/src/hotspot/share/gc/x/xMetronome.cpp
new file mode 100644
index 00000000000..7f0b649deb4
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMetronome.cpp
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xMetronome.hpp"
+#include "runtime/mutexLocker.hpp"
+#include "runtime/timer.hpp"
+#include "utilities/ticks.hpp"
+
+XMetronome::XMetronome(uint64_t hz) :
+    _monitor(Monitor::nosafepoint, "XMetronome_lock"),
+    _interval_ms(MILLIUNITS / hz),
+    _start_ms(0),
+    _nticks(0),
+    _stopped(false) {}
+
+bool XMetronome::wait_for_tick() {
+  if (_nticks++ == 0) {
+    // First tick, set start time
+    const Ticks now = Ticks::now();
+    _start_ms = TimeHelper::counter_to_millis(now.value());
+  }
+
+  MonitorLocker ml(&_monitor, Monitor::_no_safepoint_check_flag);
+
+  while (!_stopped) {
+    // We might wake up spuriously from wait, so always recalculate
+    // the timeout after a wakeup to see if we need to wait again.
+    const Ticks now = Ticks::now();
+    const uint64_t now_ms = TimeHelper::counter_to_millis(now.value());
+    const uint64_t next_ms = _start_ms + (_interval_ms * _nticks);
+    const int64_t timeout_ms = next_ms - now_ms;
+
+    if (timeout_ms > 0) {
+      // Wait
+      ml.wait(timeout_ms);
+    } else {
+      // Tick
+      if (timeout_ms < 0) {
+        const uint64_t overslept = -timeout_ms;
+        if (overslept > _interval_ms) {
+          // Missed one or more ticks. Bump _nticks accordingly to
+          // avoid firing a string of immediate ticks to make up
+          // for the ones we missed.
+          _nticks += overslept / _interval_ms;
+        }
+      }
+
+      return true;
+    }
+  }
+
+  // Stopped
+  return false;
+}
+
+void XMetronome::stop() {
+  MonitorLocker ml(&_monitor, Monitor::_no_safepoint_check_flag);
+  _stopped = true;
+  ml.notify();
+}
diff --git a/src/hotspot/share/gc/x/xMetronome.hpp b/src/hotspot/share/gc/x/xMetronome.hpp
new file mode 100644
index 00000000000..8a0f27061c3
--- /dev/null
+++ b/src/hotspot/share/gc/x/xMetronome.hpp
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XMETRONOME_HPP
+#define SHARE_GC_X_XMETRONOME_HPP
+
+#include "memory/allocation.hpp"
+#include "runtime/mutex.hpp"
+
+class XMetronome : public StackObj {
+private:
+  Monitor        _monitor;
+  const uint64_t _interval_ms;
+  uint64_t       _start_ms;
+  uint64_t       _nticks;
+  bool           _stopped;
+
+public:
+  XMetronome(uint64_t hz);
+
+  bool wait_for_tick();
+  void stop();
+};
+
+#endif // SHARE_GC_X_XMETRONOME_HPP
diff --git a/src/hotspot/share/gc/x/xNMethod.cpp b/src/hotspot/share/gc/x/xNMethod.cpp
new file mode 100644
index 00000000000..20d8982bc07
--- /dev/null
+++ b/src/hotspot/share/gc/x/xNMethod.cpp
@@ -0,0 +1,366 @@
+/*
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "code/relocInfo.hpp"
+#include "code/nmethod.hpp"
+#include "code/icBuffer.hpp"
+#include "gc/shared/barrierSet.hpp"
+#include "gc/shared/barrierSetNMethod.hpp"
+#include "gc/shared/suspendibleThreadSet.hpp"
+#include "gc/x/xBarrier.inline.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xLock.inline.hpp"
+#include "gc/x/xNMethod.hpp"
+#include "gc/x/xNMethodData.hpp"
+#include "gc/x/xNMethodTable.hpp"
+#include "gc/x/xTask.hpp"
+#include "gc/x/xWorkers.hpp"
+#include "logging/log.hpp"
+#include "memory/allocation.inline.hpp"
+#include "memory/iterator.hpp"
+#include "memory/resourceArea.hpp"
+#include "memory/universe.hpp"
+#include "oops/oop.inline.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/continuation.hpp"
+#include "utilities/debug.hpp"
+
+static XNMethodData* gc_data(const nmethod* nm) {
+  return nm->gc_data<XNMethodData>();
+}
+
+static void set_gc_data(nmethod* nm, XNMethodData* data) {
+  return nm->set_gc_data<XNMethodData>(data);
+}
+
+void XNMethod::attach_gc_data(nmethod* nm) {
+  GrowableArray<oop*> immediate_oops;
+  bool non_immediate_oops = false;
+
+  // Find all oop relocations
+  RelocIterator iter(nm);
+  while (iter.next()) {
+    if (iter.type() != relocInfo::oop_type) {
+      // Not an oop
+      continue;
+    }
+
+    oop_Relocation* r = iter.oop_reloc();
+
+    if (!r->oop_is_immediate()) {
+      // Non-immediate oop found
+      non_immediate_oops = true;
+      continue;
+    }
+
+    if (r->oop_value() != NULL) {
+      // Non-NULL immediate oop found. NULL oops can safely be
+      // ignored since the method will be re-registered if they
+      // are later patched to be non-NULL.
+      immediate_oops.push(r->oop_addr());
+    }
+  }
+
+  // Attach GC data to nmethod
+  XNMethodData* data = gc_data(nm);
+  if (data == NULL) {
+    data = new XNMethodData();
+    set_gc_data(nm, data);
+  }
+
+  // Attach oops in GC data
+  XNMethodDataOops* const new_oops = XNMethodDataOops::create(immediate_oops, non_immediate_oops);
+  XNMethodDataOops* const old_oops = data->swap_oops(new_oops);
+  XNMethodDataOops::destroy(old_oops);
+}
+
+XReentrantLock* XNMethod::lock_for_nmethod(nmethod* nm) {
+  return gc_data(nm)->lock();
+}
+
+void XNMethod::log_register(const nmethod* nm) {
+  LogTarget(Trace, gc, nmethod) log;
+  if (!log.is_enabled()) {
+    return;
+  }
+
+  const XNMethodDataOops* const oops = gc_data(nm)->oops();
+
+  log.print("Register NMethod: %s.%s (" PTR_FORMAT "), "
+            "Compiler: %s, Oops: %d, ImmediateOops: " SIZE_FORMAT ", NonImmediateOops: %s",
+            nm->method()->method_holder()->external_name(),
+            nm->method()->name()->as_C_string(),
+            p2i(nm),
+            nm->compiler_name(),
+            nm->oops_count() - 1,
+            oops->immediates_count(),
+            oops->has_non_immediates() ? "Yes" : "No");
+
+  LogTarget(Trace, gc, nmethod, oops) log_oops;
+  if (!log_oops.is_enabled()) {
+    return;
+  }
+
+  // Print nmethod oops table
+  {
+    oop* const begin = nm->oops_begin();
+    oop* const end = nm->oops_end();
+    for (oop* p = begin; p < end; p++) {
+      const oop o = Atomic::load(p); // C1 PatchingStub may replace it concurrently.
+      const char* external_name = (o == nullptr) ? "N/A" : o->klass()->external_name();
+      log_oops.print("           Oop[" SIZE_FORMAT "] " PTR_FORMAT " (%s)",
+                     (p - begin), p2i(o), external_name);
+    }
+  }
+
+  // Print nmethod immediate oops
+  {
+    oop** const begin = oops->immediates_begin();
+    oop** const end = oops->immediates_end();
+    for (oop** p = begin; p < end; p++) {
+      log_oops.print("  ImmediateOop[" SIZE_FORMAT "] " PTR_FORMAT " @ " PTR_FORMAT " (%s)",
+                     (p - begin), p2i(**p), p2i(*p), (**p)->klass()->external_name());
+    }
+  }
+}
+
+void XNMethod::log_unregister(const nmethod* nm) {
+  LogTarget(Debug, gc, nmethod) log;
+  if (!log.is_enabled()) {
+    return;
+  }
+
+  log.print("Unregister NMethod: %s.%s (" PTR_FORMAT ")",
+            nm->method()->method_holder()->external_name(),
+            nm->method()->name()->as_C_string(),
+            p2i(nm));
+}
+
+void XNMethod::register_nmethod(nmethod* nm) {
+  ResourceMark rm;
+
+  // Create and attach gc data
+  attach_gc_data(nm);
+
+  log_register(nm);
+
+  XNMethodTable::register_nmethod(nm);
+
+  // Disarm nmethod entry barrier
+  disarm(nm);
+}
+
+void XNMethod::unregister_nmethod(nmethod* nm) {
+  ResourceMark rm;
+
+  log_unregister(nm);
+
+  XNMethodTable::unregister_nmethod(nm);
+
+  // Destroy GC data
+  delete gc_data(nm);
+}
+
+bool XNMethod::supports_entry_barrier(nmethod* nm) {
+  BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod();
+  return bs->supports_entry_barrier(nm);
+}
+
+bool XNMethod::is_armed(nmethod* nm) {
+  BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod();
+  return bs->is_armed(nm);
+}
+
+void XNMethod::disarm(nmethod* nm) {
+  BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod();
+  bs->disarm(nm);
+}
+
+void XNMethod::set_guard_value(nmethod* nm, int value) {
+  BarrierSetNMethod* const bs = BarrierSet::barrier_set()->barrier_set_nmethod();
+  bs->set_guard_value(nm, value);
+}
+
+void XNMethod::nmethod_oops_do(nmethod* nm, OopClosure* cl) {
+  XLocker<XReentrantLock> locker(XNMethod::lock_for_nmethod(nm));
+  XNMethod::nmethod_oops_do_inner(nm, cl);
+}
+
+void XNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl) {
+  // Process oops table
+  {
+    oop* const begin = nm->oops_begin();
+    oop* const end = nm->oops_end();
+    for (oop* p = begin; p < end; p++) {
+      if (!Universe::contains_non_oop_word(p)) {
+        cl->do_oop(p);
+      }
+    }
+  }
+
+  XNMethodDataOops* const oops = gc_data(nm)->oops();
+
+  // Process immediate oops
+  {
+    oop** const begin = oops->immediates_begin();
+    oop** const end = oops->immediates_end();
+    for (oop** p = begin; p < end; p++) {
+      if (*p != Universe::non_oop_word()) {
+        cl->do_oop(*p);
+      }
+    }
+  }
+
+  // Process non-immediate oops
+  if (oops->has_non_immediates()) {
+    nm->fix_oop_relocations();
+  }
+}
+
+class XNMethodOopClosure : public OopClosure {
+public:
+  virtual void do_oop(oop* p) {
+    if (XResurrection::is_blocked()) {
+      XBarrier::keep_alive_barrier_on_phantom_root_oop_field(p);
+    } else {
+      XBarrier::load_barrier_on_root_oop_field(p);
+    }
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+};
+
+void XNMethod::nmethod_oops_barrier(nmethod* nm) {
+  XNMethodOopClosure cl;
+  nmethod_oops_do_inner(nm, &cl);
+}
+
+void XNMethod::nmethods_do_begin() {
+  XNMethodTable::nmethods_do_begin();
+}
+
+void XNMethod::nmethods_do_end() {
+  XNMethodTable::nmethods_do_end();
+}
+
+void XNMethod::nmethods_do(NMethodClosure* cl) {
+  XNMethodTable::nmethods_do(cl);
+}
+
+class XNMethodUnlinkClosure : public NMethodClosure {
+private:
+  bool          _unloading_occurred;
+  volatile bool _failed;
+
+  void set_failed() {
+    Atomic::store(&_failed, true);
+  }
+
+public:
+  XNMethodUnlinkClosure(bool unloading_occurred) :
+      _unloading_occurred(unloading_occurred),
+      _failed(false) {}
+
+  virtual void do_nmethod(nmethod* nm) {
+    if (failed()) {
+      return;
+    }
+
+    if (nm->is_unloading()) {
+      XLocker<XReentrantLock> locker(XNMethod::lock_for_nmethod(nm));
+      nm->unlink();
+      return;
+    }
+
+    XLocker<XReentrantLock> locker(XNMethod::lock_for_nmethod(nm));
+
+    if (XNMethod::is_armed(nm)) {
+      // Heal oops and arm phase invariantly
+      XNMethod::nmethod_oops_barrier(nm);
+      XNMethod::set_guard_value(nm, 0);
+    }
+
+    // Clear compiled ICs and exception caches
+    if (!nm->unload_nmethod_caches(_unloading_occurred)) {
+      set_failed();
+    }
+  }
+
+  bool failed() const {
+    return Atomic::load(&_failed);
+  }
+};
+
+class XNMethodUnlinkTask : public XTask {
+private:
+  XNMethodUnlinkClosure _cl;
+  ICRefillVerifier*     _verifier;
+
+public:
+  XNMethodUnlinkTask(bool unloading_occurred, ICRefillVerifier* verifier) :
+      XTask("XNMethodUnlinkTask"),
+      _cl(unloading_occurred),
+      _verifier(verifier) {
+    XNMethodTable::nmethods_do_begin();
+  }
+
+  ~XNMethodUnlinkTask() {
+    XNMethodTable::nmethods_do_end();
+  }
+
+  virtual void work() {
+    ICRefillVerifierMark mark(_verifier);
+    XNMethodTable::nmethods_do(&_cl);
+  }
+
+  bool success() const {
+    return !_cl.failed();
+  }
+};
+
+void XNMethod::unlink(XWorkers* workers, bool unloading_occurred) {
+  for (;;) {
+    ICRefillVerifier verifier;
+
+    {
+      XNMethodUnlinkTask task(unloading_occurred, &verifier);
+      workers->run(&task);
+      if (task.success()) {
+        return;
+      }
+    }
+
+    // Cleaning failed because we ran out of transitional IC stubs,
+    // so we have to refill and try again. Refilling requires taking
+    // a safepoint, so we temporarily leave the suspendible thread set.
+    SuspendibleThreadSetLeaver sts;
+    InlineCacheBuffer::refill_ic_stubs();
+  }
+}
+
+void XNMethod::purge() {
+  CodeCache::flush_unlinked_nmethods();
+}
diff --git a/src/hotspot/share/gc/x/xNMethod.hpp b/src/hotspot/share/gc/x/xNMethod.hpp
new file mode 100644
index 00000000000..cadb1601910
--- /dev/null
+++ b/src/hotspot/share/gc/x/xNMethod.hpp
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XNMETHOD_HPP
+#define SHARE_GC_X_XNMETHOD_HPP
+
+#include "memory/allStatic.hpp"
+
+class nmethod;
+class NMethodClosure;
+class XReentrantLock;
+class XWorkers;
+
+class XNMethod : public AllStatic {
+private:
+  static void attach_gc_data(nmethod* nm);
+
+  static void log_register(const nmethod* nm);
+  static void log_unregister(const nmethod* nm);
+
+public:
+  static void register_nmethod(nmethod* nm);
+  static void unregister_nmethod(nmethod* nm);
+
+  static bool supports_entry_barrier(nmethod* nm);
+
+  static bool is_armed(nmethod* nm);
+  static void disarm(nmethod* nm);
+  static void set_guard_value(nmethod* nm, int value);
+
+  static void nmethod_oops_do(nmethod* nm, OopClosure* cl);
+  static void nmethod_oops_do_inner(nmethod* nm, OopClosure* cl);
+
+  static void nmethod_oops_barrier(nmethod* nm);
+
+  static void nmethods_do_begin();
+  static void nmethods_do_end();
+  static void nmethods_do(NMethodClosure* cl);
+
+  static XReentrantLock* lock_for_nmethod(nmethod* nm);
+
+  static void unlink(XWorkers* workers, bool unloading_occurred);
+  static void purge();
+};
+
+#endif // SHARE_GC_X_XNMETHOD_HPP
diff --git a/src/hotspot/share/gc/x/xNMethodData.cpp b/src/hotspot/share/gc/x/xNMethodData.cpp
new file mode 100644
index 00000000000..f024e0d3cee
--- /dev/null
+++ b/src/hotspot/share/gc/x/xNMethodData.cpp
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xAttachedArray.inline.hpp"
+#include "gc/x/xLock.inline.hpp"
+#include "gc/x/xNMethodData.hpp"
+#include "memory/allocation.hpp"
+#include "runtime/atomic.hpp"
+#include "utilities/align.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/growableArray.hpp"
+
+XNMethodDataOops* XNMethodDataOops::create(const GrowableArray<oop*>& immediates, bool has_non_immediates) {
+  return ::new (AttachedArray::alloc(immediates.length())) XNMethodDataOops(immediates, has_non_immediates);
+}
+
+void XNMethodDataOops::destroy(XNMethodDataOops* oops) {
+  AttachedArray::free(oops);
+}
+
+XNMethodDataOops::XNMethodDataOops(const GrowableArray<oop*>& immediates, bool has_non_immediates) :
+    _immediates(immediates.length()),
+    _has_non_immediates(has_non_immediates) {
+  // Save all immediate oops
+  for (size_t i = 0; i < immediates_count(); i++) {
+    immediates_begin()[i] = immediates.at(int(i));
+  }
+}
+
+size_t XNMethodDataOops::immediates_count() const {
+  return _immediates.length();
+}
+
+oop** XNMethodDataOops::immediates_begin() const {
+  return _immediates(this);
+}
+
+oop** XNMethodDataOops::immediates_end() const {
+  return immediates_begin() + immediates_count();
+}
+
+bool XNMethodDataOops::has_non_immediates() const {
+  return _has_non_immediates;
+}
+
+XNMethodData::XNMethodData() :
+    _lock(),
+    _oops(NULL) {}
+
+XNMethodData::~XNMethodData() {
+  XNMethodDataOops::destroy(_oops);
+}
+
+XReentrantLock* XNMethodData::lock() {
+  return &_lock;
+}
+
+XNMethodDataOops* XNMethodData::oops() const {
+  return Atomic::load_acquire(&_oops);
+}
+
+XNMethodDataOops* XNMethodData::swap_oops(XNMethodDataOops* new_oops) {
+  XLocker<XReentrantLock> locker(&_lock);
+  XNMethodDataOops* const old_oops = _oops;
+  _oops = new_oops;
+  return old_oops;
+}
diff --git a/src/hotspot/share/gc/x/xNMethodData.hpp b/src/hotspot/share/gc/x/xNMethodData.hpp
new file mode 100644
index 00000000000..f0e45ef5d14
--- /dev/null
+++ b/src/hotspot/share/gc/x/xNMethodData.hpp
@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XNMETHODDATA_HPP
+#define SHARE_GC_X_XNMETHODDATA_HPP
+
+#include "gc/x/xAttachedArray.hpp"
+#include "gc/x/xLock.hpp"
+#include "memory/allocation.hpp"
+#include "oops/oopsHierarchy.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class nmethod;
+template <typename T> class GrowableArray;
+
+class XNMethodDataOops {
+private:
+  typedef XAttachedArray<XNMethodDataOops, oop*> AttachedArray;
+
+  const AttachedArray _immediates;
+  const bool          _has_non_immediates;
+
+  XNMethodDataOops(const GrowableArray<oop*>& immediates, bool has_non_immediates);
+
+public:
+  static XNMethodDataOops* create(const GrowableArray<oop*>& immediates, bool has_non_immediates);
+  static void destroy(XNMethodDataOops* oops);
+
+  size_t immediates_count() const;
+  oop** immediates_begin() const;
+  oop** immediates_end() const;
+
+  bool has_non_immediates() const;
+};
+
+class XNMethodData : public CHeapObj<mtGC> {
+private:
+  XReentrantLock             _lock;
+  XNMethodDataOops* volatile _oops;
+
+public:
+  XNMethodData();
+  ~XNMethodData();
+
+  XReentrantLock* lock();
+
+  XNMethodDataOops* oops() const;
+  XNMethodDataOops* swap_oops(XNMethodDataOops* oops);
+};
+
+#endif // SHARE_GC_X_XNMETHODDATA_HPP
diff --git a/src/hotspot/share/gc/x/xNMethodTable.cpp b/src/hotspot/share/gc/x/xNMethodTable.cpp
new file mode 100644
index 00000000000..70ceb7a9219
--- /dev/null
+++ b/src/hotspot/share/gc/x/xNMethodTable.cpp
@@ -0,0 +1,235 @@
+/*
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "code/relocInfo.hpp"
+#include "code/nmethod.hpp"
+#include "code/icBuffer.hpp"
+#include "gc/shared/barrierSet.hpp"
+#include "gc/shared/barrierSetNMethod.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xHash.inline.hpp"
+#include "gc/x/xLock.inline.hpp"
+#include "gc/x/xNMethodData.hpp"
+#include "gc/x/xNMethodTable.hpp"
+#include "gc/x/xNMethodTableEntry.hpp"
+#include "gc/x/xNMethodTableIteration.hpp"
+#include "gc/x/xSafeDelete.inline.hpp"
+#include "gc/x/xTask.hpp"
+#include "gc/x/xWorkers.hpp"
+#include "logging/log.hpp"
+#include "memory/allocation.hpp"
+#include "memory/iterator.hpp"
+#include "memory/resourceArea.hpp"
+#include "runtime/mutexLocker.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+XNMethodTableEntry* XNMethodTable::_table = NULL;
+size_t XNMethodTable::_size = 0;
+size_t XNMethodTable::_nregistered = 0;
+size_t XNMethodTable::_nunregistered = 0;
+XNMethodTableIteration XNMethodTable::_iteration;
+XSafeDeleteNoLock<XNMethodTableEntry[]> XNMethodTable::_safe_delete;
+
+size_t XNMethodTable::first_index(const nmethod* nm, size_t size) {
+  assert(is_power_of_2(size), "Invalid size");
+  const size_t mask = size - 1;
+  const size_t hash = XHash::address_to_uint32((uintptr_t)nm);
+  return hash & mask;
+}
+
+size_t XNMethodTable::next_index(size_t prev_index, size_t size) {
+  assert(is_power_of_2(size), "Invalid size");
+  const size_t mask = size - 1;
+  return (prev_index + 1) & mask;
+}
+
+bool XNMethodTable::register_entry(XNMethodTableEntry* table, size_t size, nmethod* nm) {
+  const XNMethodTableEntry entry(nm);
+  size_t index = first_index(nm, size);
+
+  for (;;) {
+    const XNMethodTableEntry table_entry = table[index];
+
+    if (!table_entry.registered() && !table_entry.unregistered()) {
+      // Insert new entry
+      table[index] = entry;
+      return true;
+    }
+
+    if (table_entry.registered() && table_entry.method() == nm) {
+      // Replace existing entry
+      table[index] = entry;
+      return false;
+    }
+
+    index = next_index(index, size);
+  }
+}
+
+void XNMethodTable::unregister_entry(XNMethodTableEntry* table, size_t size, nmethod* nm) {
+  size_t index = first_index(nm, size);
+
+  for (;;) {
+    const XNMethodTableEntry table_entry = table[index];
+    assert(table_entry.registered() || table_entry.unregistered(), "Entry not found");
+
+    if (table_entry.registered() && table_entry.method() == nm) {
+      // Remove entry
+      table[index] = XNMethodTableEntry(true /* unregistered */);
+      return;
+    }
+
+    index = next_index(index, size);
+  }
+}
+
+void XNMethodTable::rebuild(size_t new_size) {
+  assert(CodeCache_lock->owned_by_self(), "Lock must be held");
+
+  assert(is_power_of_2(new_size), "Invalid size");
+
+  log_debug(gc, nmethod)("Rebuilding NMethod Table: "
+                         SIZE_FORMAT "->" SIZE_FORMAT " entries, "
+                         SIZE_FORMAT "(%.0f%%->%.0f%%) registered, "
+                         SIZE_FORMAT "(%.0f%%->%.0f%%) unregistered",
+                         _size, new_size,
+                         _nregistered, percent_of(_nregistered, _size), percent_of(_nregistered, new_size),
+                         _nunregistered, percent_of(_nunregistered, _size), 0.0);
+
+  // Allocate new table
+  XNMethodTableEntry* const new_table = new XNMethodTableEntry[new_size];
+
+  // Transfer all registered entries
+  for (size_t i = 0; i < _size; i++) {
+    const XNMethodTableEntry entry = _table[i];
+    if (entry.registered()) {
+      register_entry(new_table, new_size, entry.method());
+    }
+  }
+
+  // Free old table
+  _safe_delete(_table);
+
+  // Install new table
+  _table = new_table;
+  _size = new_size;
+  _nunregistered = 0;
+}
+
+void XNMethodTable::rebuild_if_needed() {
+  // The hash table uses linear probing. To avoid wasting memory while
+  // at the same time maintaining good hash collision behavior we want
+  // to keep the table occupancy between 30% and 70%. The table always
+  // grows/shrinks by doubling/halving its size. Pruning of unregistered
+  // entries is done by rebuilding the table with or without resizing it.
+  const size_t min_size = 1024;
+  const size_t shrink_threshold = _size * 0.30;
+  const size_t prune_threshold = _size * 0.65;
+  const size_t grow_threshold = _size * 0.70;
+
+  if (_size == 0) {
+    // Initialize table
+    rebuild(min_size);
+  } else if (_nregistered < shrink_threshold && _size > min_size) {
+    // Shrink table
+    rebuild(_size / 2);
+  } else if (_nregistered + _nunregistered > grow_threshold) {
+    // Prune or grow table
+    if (_nregistered < prune_threshold) {
+      // Prune table
+      rebuild(_size);
+    } else {
+      // Grow table
+      rebuild(_size * 2);
+    }
+  }
+}
+
+size_t XNMethodTable::registered_nmethods() {
+  return _nregistered;
+}
+
+size_t XNMethodTable::unregistered_nmethods() {
+  return _nunregistered;
+}
+
+void XNMethodTable::register_nmethod(nmethod* nm) {
+  assert(CodeCache_lock->owned_by_self(), "Lock must be held");
+
+  // Grow/Shrink/Prune table if needed
+  rebuild_if_needed();
+
+  // Insert new entry
+  if (register_entry(_table, _size, nm)) {
+    // New entry registered. When register_entry() instead returns
+    // false the nmethod was already in the table so we do not want
+    // to increase number of registered entries in that case.
+    _nregistered++;
+  }
+}
+
+void XNMethodTable::wait_until_iteration_done() {
+  assert(CodeCache_lock->owned_by_self(), "Lock must be held");
+
+  while (_iteration.in_progress()) {
+    CodeCache_lock->wait_without_safepoint_check();
+  }
+}
+
+void XNMethodTable::unregister_nmethod(nmethod* nm) {
+  assert(CodeCache_lock->owned_by_self(), "Lock must be held");
+
+  // Remove entry
+  unregister_entry(_table, _size, nm);
+  _nunregistered++;
+  _nregistered--;
+}
+
+void XNMethodTable::nmethods_do_begin() {
+  MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
+
+  // Do not allow the table to be deleted while iterating
+  _safe_delete.enable_deferred_delete();
+
+  // Prepare iteration
+  _iteration.nmethods_do_begin(_table, _size);
+}
+
+void XNMethodTable::nmethods_do_end() {
+  MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
+
+  // Finish iteration
+  _iteration.nmethods_do_end();
+
+  // Allow the table to be deleted
+  _safe_delete.disable_deferred_delete();
+
+  // Notify iteration done
+  CodeCache_lock->notify_all();
+}
+
+void XNMethodTable::nmethods_do(NMethodClosure* cl) {
+  _iteration.nmethods_do(cl);
+}
diff --git a/src/hotspot/share/gc/x/xNMethodTable.hpp b/src/hotspot/share/gc/x/xNMethodTable.hpp
new file mode 100644
index 00000000000..ebb7803a083
--- /dev/null
+++ b/src/hotspot/share/gc/x/xNMethodTable.hpp
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XNMETHODTABLE_HPP
+#define SHARE_GC_X_XNMETHODTABLE_HPP
+
+#include "gc/x/xNMethodTableIteration.hpp"
+#include "gc/x/xSafeDelete.hpp"
+#include "memory/allStatic.hpp"
+
+class nmethod;
+class NMethodClosure;
+class XNMethodTableEntry;
+class XWorkers;
+
+class XNMethodTable : public AllStatic {
+private:
+  static XNMethodTableEntry*                     _table;
+  static size_t                                  _size;
+  static size_t                                  _nregistered;
+  static size_t                                  _nunregistered;
+  static XNMethodTableIteration                  _iteration;
+  static XSafeDeleteNoLock<XNMethodTableEntry[]> _safe_delete;
+
+  static XNMethodTableEntry* create(size_t size);
+  static void destroy(XNMethodTableEntry* table);
+
+  static size_t first_index(const nmethod* nm, size_t size);
+  static size_t next_index(size_t prev_index, size_t size);
+
+  static bool register_entry(XNMethodTableEntry* table, size_t size, nmethod* nm);
+  static void unregister_entry(XNMethodTableEntry* table, size_t size, nmethod* nm);
+
+  static void rebuild(size_t new_size);
+  static void rebuild_if_needed();
+
+public:
+  static size_t registered_nmethods();
+  static size_t unregistered_nmethods();
+
+  static void register_nmethod(nmethod* nm);
+  static void unregister_nmethod(nmethod* nm);
+
+  static void wait_until_iteration_done();
+
+  static void nmethods_do_begin();
+  static void nmethods_do_end();
+  static void nmethods_do(NMethodClosure* cl);
+
+  static void unlink(XWorkers* workers, bool unloading_occurred);
+  static void purge(XWorkers* workers);
+};
+
+#endif // SHARE_GC_X_XNMETHODTABLE_HPP
diff --git a/src/hotspot/share/gc/x/xNMethodTableEntry.hpp b/src/hotspot/share/gc/x/xNMethodTableEntry.hpp
new file mode 100644
index 00000000000..78138492ef9
--- /dev/null
+++ b/src/hotspot/share/gc/x/xNMethodTableEntry.hpp
@@ -0,0 +1,81 @@
+/*
+ * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XNMETHODTABLEENTRY_HPP
+#define SHARE_GC_X_XNMETHODTABLEENTRY_HPP
+
+#include "gc/x/xBitField.hpp"
+#include "memory/allocation.hpp"
+
+class nmethod;
+
+//
+// NMethod table entry layout
+// --------------------------
+//
+//   6
+//   3                                                                   2 1 0
+//  +---------------------------------------------------------------------+-+-+
+//  |11111111 11111111 11111111 11111111 11111111 11111111 11111111 111111|1|1|
+//  +---------------------------------------------------------------------+-+-+
+//  |                                                                     | |
+//  |                                      1-1 Unregistered Flag (1-bits) * |
+//  |                                                                       |
+//  |                                          0-0 Registered Flag (1-bits) *
+//  |
+//  * 63-2 NMethod Address (62-bits)
+//
+
+class XNMethodTableEntry : public CHeapObj<mtGC> {
+private:
+  typedef XBitField<uint64_t, bool,     0,  1>    field_registered;
+  typedef XBitField<uint64_t, bool,     1,  1>    field_unregistered;
+  typedef XBitField<uint64_t, nmethod*, 2, 62, 2> field_method;
+
+  uint64_t _entry;
+
+public:
+  explicit XNMethodTableEntry(bool unregistered = false) :
+      _entry(field_registered::encode(false) |
+             field_unregistered::encode(unregistered) |
+             field_method::encode(NULL)) {}
+
+  explicit XNMethodTableEntry(nmethod* method) :
+      _entry(field_registered::encode(true) |
+             field_unregistered::encode(false) |
+             field_method::encode(method)) {}
+
+  bool registered() const {
+    return field_registered::decode(_entry);
+  }
+
+  bool unregistered() const {
+    return field_unregistered::decode(_entry);
+  }
+
+  nmethod* method() const {
+    return field_method::decode(_entry);
+  }
+};
+
+#endif // SHARE_GC_X_XNMETHODTABLEENTRY_HPP
diff --git a/src/hotspot/share/gc/x/xNMethodTableIteration.cpp b/src/hotspot/share/gc/x/xNMethodTableIteration.cpp
new file mode 100644
index 00000000000..4487cc4e51f
--- /dev/null
+++ b/src/hotspot/share/gc/x/xNMethodTableIteration.cpp
@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xNMethodTableEntry.hpp"
+#include "gc/x/xNMethodTableIteration.hpp"
+#include "memory/iterator.hpp"
+#include "runtime/atomic.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+XNMethodTableIteration::XNMethodTableIteration() :
+    _table(NULL),
+    _size(0),
+    _claimed(0) {}
+
+bool XNMethodTableIteration::in_progress() const {
+  return _table != NULL;
+}
+
+void XNMethodTableIteration::nmethods_do_begin(XNMethodTableEntry* table, size_t size) {
+  assert(!in_progress(), "precondition");
+
+  _table = table;
+  _size = size;
+  _claimed = 0;
+}
+
+void XNMethodTableIteration::nmethods_do_end() {
+  assert(_claimed >= _size, "Failed to claim all table entries");
+
+  // Finish iteration
+  _table = NULL;
+}
+
+void XNMethodTableIteration::nmethods_do(NMethodClosure* cl) {
+  for (;;) {
+    // Claim table partition. Each partition is currently sized to span
+    // two cache lines. This number is just a guess, but seems to work well.
+    const size_t partition_size = (XCacheLineSize * 2) / sizeof(XNMethodTableEntry);
+    const size_t partition_start = MIN2(Atomic::fetch_and_add(&_claimed, partition_size), _size);
+    const size_t partition_end = MIN2(partition_start + partition_size, _size);
+    if (partition_start == partition_end) {
+      // End of table
+      break;
+    }
+
+    // Process table partition
+    for (size_t i = partition_start; i < partition_end; i++) {
+      const XNMethodTableEntry entry = _table[i];
+      if (entry.registered()) {
+        cl->do_nmethod(entry.method());
+      }
+    }
+  }
+}
diff --git a/src/hotspot/share/gc/x/xNMethodTableIteration.hpp b/src/hotspot/share/gc/x/xNMethodTableIteration.hpp
new file mode 100644
index 00000000000..1677b334490
--- /dev/null
+++ b/src/hotspot/share/gc/x/xNMethodTableIteration.hpp
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XNMETHODTABLEITERATION_HPP
+#define SHARE_GC_X_XNMETHODTABLEITERATION_HPP
+
+#include "gc/x/xGlobals.hpp"
+
+class NMethodClosure;
+class XNMethodTableEntry;
+
+class XNMethodTableIteration {
+private:
+  XNMethodTableEntry*            _table;
+  size_t                         _size;
+  XCACHE_ALIGNED volatile size_t _claimed;
+
+public:
+  XNMethodTableIteration();
+
+  bool in_progress() const;
+
+  void nmethods_do_begin(XNMethodTableEntry* table, size_t size);
+  void nmethods_do_end();
+  void nmethods_do(NMethodClosure* cl);
+};
+
+#endif // SHARE_GC_X_XNMETHODTABLEITERATION_HPP
diff --git a/src/hotspot/share/gc/x/xNUMA.cpp b/src/hotspot/share/gc/x/xNUMA.cpp
new file mode 100644
index 00000000000..fb99878b200
--- /dev/null
+++ b/src/hotspot/share/gc/x/xNUMA.cpp
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/x/xNUMA.hpp"
+
+bool XNUMA::_enabled;
+
+void XNUMA::initialize() {
+  pd_initialize();
+
+  log_info_p(gc, init)("NUMA Support: %s", to_string());
+  if (_enabled) {
+    log_info_p(gc, init)("NUMA Nodes: %u", count());
+  }
+}
+
+const char* XNUMA::to_string() {
+  return _enabled ? "Enabled" : "Disabled";
+}
diff --git a/src/hotspot/share/gc/x/xNUMA.hpp b/src/hotspot/share/gc/x/xNUMA.hpp
new file mode 100644
index 00000000000..6331a62c042
--- /dev/null
+++ b/src/hotspot/share/gc/x/xNUMA.hpp
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XNUMA_HPP
+#define SHARE_GC_X_XNUMA_HPP
+
+#include "memory/allStatic.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class XNUMA : public AllStatic {
+private:
+  static bool _enabled;
+
+  static void pd_initialize();
+
+public:
+  static void initialize();
+  static bool is_enabled();
+
+  static uint32_t count();
+  static uint32_t id();
+
+  static uint32_t memory_id(uintptr_t addr);
+
+  static const char* to_string();
+};
+
+#endif // SHARE_GC_X_XNUMA_HPP
diff --git a/src/hotspot/share/gc/x/xNUMA.inline.hpp b/src/hotspot/share/gc/x/xNUMA.inline.hpp
new file mode 100644
index 00000000000..17f5b831a31
--- /dev/null
+++ b/src/hotspot/share/gc/x/xNUMA.inline.hpp
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XNUMA_INLINE_HPP
+#define SHARE_GC_X_XNUMA_INLINE_HPP
+
+#include "gc/x/xNUMA.hpp"
+
+inline bool XNUMA::is_enabled() {
+  return _enabled;
+}
+
+#endif // SHARE_GC_X_XNUMA_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xObjArrayAllocator.cpp b/src/hotspot/share/gc/x/xObjArrayAllocator.cpp
new file mode 100644
index 00000000000..9408e027cbd
--- /dev/null
+++ b/src/hotspot/share/gc/x/xObjArrayAllocator.cpp
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+#include "gc/x/xObjArrayAllocator.hpp"
+#include "gc/x/xUtils.inline.hpp"
+#include "oops/arrayKlass.hpp"
+#include "runtime/interfaceSupport.inline.hpp"
+#include "utilities/debug.hpp"
+
+XObjArrayAllocator::XObjArrayAllocator(Klass* klass, size_t word_size, int length, bool do_zero, Thread* thread) :
+    ObjArrayAllocator(klass, word_size, length, do_zero, thread) {}
+
+void XObjArrayAllocator::yield_for_safepoint() const {
+  ThreadBlockInVM tbivm(JavaThread::cast(_thread));
+}
+
+oop XObjArrayAllocator::initialize(HeapWord* mem) const {
+  // ZGC specializes the initialization by performing segmented clearing
+  // to allow shorter time-to-safepoints.
+
+  if (!_do_zero) {
+    // No need for ZGC specialization
+    return ObjArrayAllocator::initialize(mem);
+  }
+
+  // A max segment size of 64K was chosen because microbenchmarking
+  // suggested that it offered a good trade-off between allocation
+  // time and time-to-safepoint
+  const size_t segment_max = XUtils::bytes_to_words(64 * K);
+  const BasicType element_type = ArrayKlass::cast(_klass)->element_type();
+  const size_t header = arrayOopDesc::header_size(element_type);
+  const size_t payload_size = _word_size - header;
+
+  if (payload_size <= segment_max) {
+    // To small to use segmented clearing
+    return ObjArrayAllocator::initialize(mem);
+  }
+
+  // Segmented clearing
+
+  // The array is going to be exposed before it has been completely
+  // cleared, therefore we can't expose the header at the end of this
+  // function. Instead explicitly initialize it according to our needs.
+  arrayOopDesc::set_mark(mem, markWord::prototype());
+  arrayOopDesc::release_set_klass(mem, _klass);
+  assert(_length >= 0, "length should be non-negative");
+  arrayOopDesc::set_length(mem, _length);
+
+  // Keep the array alive across safepoints through an invisible
+  // root. Invisible roots are not visited by the heap itarator
+  // and the marking logic will not attempt to follow its elements.
+  // Relocation knows how to dodge iterating over such objects.
+  XThreadLocalData::set_invisible_root(_thread, (oop*)&mem);
+
+  for (size_t processed = 0; processed < payload_size; processed += segment_max) {
+    // Calculate segment
+    HeapWord* const start = (HeapWord*)(mem + header + processed);
+    const size_t remaining = payload_size - processed;
+    const size_t segment_size = MIN2(remaining, segment_max);
+
+    // Clear segment
+    Copy::zero_to_words(start, segment_size);
+
+    // Safepoint
+    yield_for_safepoint();
+  }
+
+  XThreadLocalData::clear_invisible_root(_thread);
+
+  return cast_to_oop(mem);
+}
diff --git a/src/hotspot/share/gc/x/xObjArrayAllocator.hpp b/src/hotspot/share/gc/x/xObjArrayAllocator.hpp
new file mode 100644
index 00000000000..4a084da3279
--- /dev/null
+++ b/src/hotspot/share/gc/x/xObjArrayAllocator.hpp
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XOBJARRAYALLOCATOR_HPP
+#define SHARE_GC_X_XOBJARRAYALLOCATOR_HPP
+
+#include "gc/shared/memAllocator.hpp"
+
+class XObjArrayAllocator : public ObjArrayAllocator {
+private:
+  virtual oop initialize(HeapWord* mem) const override;
+
+  void yield_for_safepoint() const;
+
+public:
+  XObjArrayAllocator(Klass* klass, size_t word_size, int length, bool do_zero, Thread* thread);
+};
+
+#endif // SHARE_GC_X_XOBJARRAYALLOCATOR_HPP
diff --git a/src/hotspot/share/gc/x/xObjectAllocator.cpp b/src/hotspot/share/gc/x/xObjectAllocator.cpp
new file mode 100644
index 00000000000..589e2f2feba
--- /dev/null
+++ b/src/hotspot/share/gc/x/xObjectAllocator.cpp
@@ -0,0 +1,267 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xHeap.inline.hpp"
+#include "gc/x/xHeuristics.hpp"
+#include "gc/x/xObjectAllocator.hpp"
+#include "gc/x/xPage.inline.hpp"
+#include "gc/x/xPageTable.inline.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xThread.inline.hpp"
+#include "gc/x/xValue.inline.hpp"
+#include "logging/log.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/safepoint.hpp"
+#include "utilities/align.hpp"
+#include "utilities/debug.hpp"
+
+static const XStatCounter XCounterUndoObjectAllocationSucceeded("Memory", "Undo Object Allocation Succeeded", XStatUnitOpsPerSecond);
+static const XStatCounter XCounterUndoObjectAllocationFailed("Memory", "Undo Object Allocation Failed", XStatUnitOpsPerSecond);
+
+XObjectAllocator::XObjectAllocator() :
+    _use_per_cpu_shared_small_pages(XHeuristics::use_per_cpu_shared_small_pages()),
+    _used(0),
+    _undone(0),
+    _alloc_for_relocation(0),
+    _undo_alloc_for_relocation(0),
+    _shared_medium_page(NULL),
+    _shared_small_page(NULL) {}
+
+XPage** XObjectAllocator::shared_small_page_addr() {
+  return _use_per_cpu_shared_small_pages ? _shared_small_page.addr() : _shared_small_page.addr(0);
+}
+
+XPage* const* XObjectAllocator::shared_small_page_addr() const {
+  return _use_per_cpu_shared_small_pages ? _shared_small_page.addr() : _shared_small_page.addr(0);
+}
+
+void XObjectAllocator::register_alloc_for_relocation(const XPageTable* page_table, uintptr_t addr, size_t size) {
+  const XPage* const page = page_table->get(addr);
+  const size_t aligned_size = align_up(size, page->object_alignment());
+  Atomic::add(_alloc_for_relocation.addr(), aligned_size);
+}
+
+void XObjectAllocator::register_undo_alloc_for_relocation(const XPage* page, size_t size) {
+  const size_t aligned_size = align_up(size, page->object_alignment());
+  Atomic::add(_undo_alloc_for_relocation.addr(), aligned_size);
+}
+
+XPage* XObjectAllocator::alloc_page(uint8_t type, size_t size, XAllocationFlags flags) {
+  XPage* const page = XHeap::heap()->alloc_page(type, size, flags);
+  if (page != NULL) {
+    // Increment used bytes
+    Atomic::add(_used.addr(), size);
+  }
+
+  return page;
+}
+
+void XObjectAllocator::undo_alloc_page(XPage* page) {
+  // Increment undone bytes
+  Atomic::add(_undone.addr(), page->size());
+
+  XHeap::heap()->undo_alloc_page(page);
+}
+
+uintptr_t XObjectAllocator::alloc_object_in_shared_page(XPage** shared_page,
+                                                        uint8_t page_type,
+                                                        size_t page_size,
+                                                        size_t size,
+                                                        XAllocationFlags flags) {
+  uintptr_t addr = 0;
+  XPage* page = Atomic::load_acquire(shared_page);
+
+  if (page != NULL) {
+    addr = page->alloc_object_atomic(size);
+  }
+
+  if (addr == 0) {
+    // Allocate new page
+    XPage* const new_page = alloc_page(page_type, page_size, flags);
+    if (new_page != NULL) {
+      // Allocate object before installing the new page
+      addr = new_page->alloc_object(size);
+
+    retry:
+      // Install new page
+      XPage* const prev_page = Atomic::cmpxchg(shared_page, page, new_page);
+      if (prev_page != page) {
+        if (prev_page == NULL) {
+          // Previous page was retired, retry installing the new page
+          page = prev_page;
+          goto retry;
+        }
+
+        // Another page already installed, try allocation there first
+        const uintptr_t prev_addr = prev_page->alloc_object_atomic(size);
+        if (prev_addr == 0) {
+          // Allocation failed, retry installing the new page
+          page = prev_page;
+          goto retry;
+        }
+
+        // Allocation succeeded in already installed page
+        addr = prev_addr;
+
+        // Undo new page allocation
+        undo_alloc_page(new_page);
+      }
+    }
+  }
+
+  return addr;
+}
+
+uintptr_t XObjectAllocator::alloc_large_object(size_t size, XAllocationFlags flags) {
+  uintptr_t addr = 0;
+
+  // Allocate new large page
+  const size_t page_size = align_up(size, XGranuleSize);
+  XPage* const page = alloc_page(XPageTypeLarge, page_size, flags);
+  if (page != NULL) {
+    // Allocate the object
+    addr = page->alloc_object(size);
+  }
+
+  return addr;
+}
+
+uintptr_t XObjectAllocator::alloc_medium_object(size_t size, XAllocationFlags flags) {
+  return alloc_object_in_shared_page(_shared_medium_page.addr(), XPageTypeMedium, XPageSizeMedium, size, flags);
+}
+
+uintptr_t XObjectAllocator::alloc_small_object(size_t size, XAllocationFlags flags) {
+  return alloc_object_in_shared_page(shared_small_page_addr(), XPageTypeSmall, XPageSizeSmall, size, flags);
+}
+
+uintptr_t XObjectAllocator::alloc_object(size_t size, XAllocationFlags flags) {
+  if (size <= XObjectSizeLimitSmall) {
+    // Small
+    return alloc_small_object(size, flags);
+  } else if (size <= XObjectSizeLimitMedium) {
+    // Medium
+    return alloc_medium_object(size, flags);
+  } else {
+    // Large
+    return alloc_large_object(size, flags);
+  }
+}
+
+uintptr_t XObjectAllocator::alloc_object(size_t size) {
+  XAllocationFlags flags;
+  return alloc_object(size, flags);
+}
+
+uintptr_t XObjectAllocator::alloc_object_for_relocation(const XPageTable* page_table, size_t size) {
+  XAllocationFlags flags;
+  flags.set_non_blocking();
+
+  const uintptr_t addr = alloc_object(size, flags);
+  if (addr != 0) {
+    register_alloc_for_relocation(page_table, addr, size);
+  }
+
+  return addr;
+}
+
+void XObjectAllocator::undo_alloc_object_for_relocation(XPage* page, uintptr_t addr, size_t size) {
+  const uint8_t type = page->type();
+
+  if (type == XPageTypeLarge) {
+    register_undo_alloc_for_relocation(page, size);
+    undo_alloc_page(page);
+    XStatInc(XCounterUndoObjectAllocationSucceeded);
+  } else {
+    if (page->undo_alloc_object_atomic(addr, size)) {
+      register_undo_alloc_for_relocation(page, size);
+      XStatInc(XCounterUndoObjectAllocationSucceeded);
+    } else {
+      XStatInc(XCounterUndoObjectAllocationFailed);
+    }
+  }
+}
+
+size_t XObjectAllocator::used() const {
+  size_t total_used = 0;
+  size_t total_undone = 0;
+
+  XPerCPUConstIterator<size_t> iter_used(&_used);
+  for (const size_t* cpu_used; iter_used.next(&cpu_used);) {
+    total_used += *cpu_used;
+  }
+
+  XPerCPUConstIterator<size_t> iter_undone(&_undone);
+  for (const size_t* cpu_undone; iter_undone.next(&cpu_undone);) {
+    total_undone += *cpu_undone;
+  }
+
+  return total_used - total_undone;
+}
+
+size_t XObjectAllocator::remaining() const {
+  assert(XThread::is_java(), "Should be a Java thread");
+
+  const XPage* const page = Atomic::load_acquire(shared_small_page_addr());
+  if (page != NULL) {
+    return page->remaining();
+  }
+
+  return 0;
+}
+
+size_t XObjectAllocator::relocated() const {
+  size_t total_alloc = 0;
+  size_t total_undo_alloc = 0;
+
+  XPerCPUConstIterator<size_t> iter_alloc(&_alloc_for_relocation);
+  for (const size_t* alloc; iter_alloc.next(&alloc);) {
+    total_alloc += Atomic::load(alloc);
+  }
+
+  XPerCPUConstIterator<size_t> iter_undo_alloc(&_undo_alloc_for_relocation);
+  for (const size_t* undo_alloc; iter_undo_alloc.next(&undo_alloc);) {
+    total_undo_alloc += Atomic::load(undo_alloc);
+  }
+
+  assert(total_alloc >= total_undo_alloc, "Mismatch");
+
+  return total_alloc - total_undo_alloc;
+}
+
+void XObjectAllocator::retire_pages() {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+
+  // Reset used and undone bytes
+  _used.set_all(0);
+  _undone.set_all(0);
+
+  // Reset relocated bytes
+  _alloc_for_relocation.set_all(0);
+  _undo_alloc_for_relocation.set_all(0);
+
+  // Reset allocation pages
+  _shared_medium_page.set(NULL);
+  _shared_small_page.set_all(NULL);
+}
diff --git a/src/hotspot/share/gc/x/xObjectAllocator.hpp b/src/hotspot/share/gc/x/xObjectAllocator.hpp
new file mode 100644
index 00000000000..8880c41f3d5
--- /dev/null
+++ b/src/hotspot/share/gc/x/xObjectAllocator.hpp
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XOBJECTALLOCATOR_HPP
+#define SHARE_GC_X_XOBJECTALLOCATOR_HPP
+
+#include "gc/x/xAllocationFlags.hpp"
+#include "gc/x/xValue.hpp"
+
+class XPage;
+class XPageTable;
+
+class XObjectAllocator {
+private:
+  const bool         _use_per_cpu_shared_small_pages;
+  XPerCPU<size_t>    _used;
+  XPerCPU<size_t>    _undone;
+  XPerCPU<size_t>    _alloc_for_relocation;
+  XPerCPU<size_t>    _undo_alloc_for_relocation;
+  XContended<XPage*> _shared_medium_page;
+  XPerCPU<XPage*>    _shared_small_page;
+
+  XPage** shared_small_page_addr();
+  XPage* const* shared_small_page_addr() const;
+
+  void register_alloc_for_relocation(const XPageTable* page_table, uintptr_t addr, size_t size);
+  void register_undo_alloc_for_relocation(const XPage* page, size_t size);
+
+  XPage* alloc_page(uint8_t type, size_t size, XAllocationFlags flags);
+  void undo_alloc_page(XPage* page);
+
+  // Allocate an object in a shared page. Allocate and
+  // atomically install a new page if necessary.
+  uintptr_t alloc_object_in_shared_page(XPage** shared_page,
+                                        uint8_t page_type,
+                                        size_t page_size,
+                                        size_t size,
+                                        XAllocationFlags flags);
+
+  uintptr_t alloc_large_object(size_t size, XAllocationFlags flags);
+  uintptr_t alloc_medium_object(size_t size, XAllocationFlags flags);
+  uintptr_t alloc_small_object(size_t size, XAllocationFlags flags);
+  uintptr_t alloc_object(size_t size, XAllocationFlags flags);
+
+public:
+  XObjectAllocator();
+
+  uintptr_t alloc_object(size_t size);
+  uintptr_t alloc_object_for_relocation(const XPageTable* page_table, size_t size);
+  void undo_alloc_object_for_relocation(XPage* page, uintptr_t addr, size_t size);
+
+  size_t used() const;
+  size_t remaining() const;
+  size_t relocated() const;
+
+  void retire_pages();
+};
+
+#endif // SHARE_GC_X_XOBJECTALLOCATOR_HPP
diff --git a/src/hotspot/share/gc/z/zOop.hpp b/src/hotspot/share/gc/x/xOop.hpp
similarity index 91%
rename from src/hotspot/share/gc/z/zOop.hpp
rename to src/hotspot/share/gc/x/xOop.hpp
index 4fb0e6499e1..92cc7a225fe 100644
--- a/src/hotspot/share/gc/z/zOop.hpp
+++ b/src/hotspot/share/gc/x/xOop.hpp
@@ -21,16 +21,16 @@
  * questions.
  */
 
-#ifndef SHARE_GC_Z_ZOOP_HPP
-#define SHARE_GC_Z_ZOOP_HPP
+#ifndef SHARE_GC_X_XOOP_HPP
+#define SHARE_GC_X_XOOP_HPP
 
 #include "memory/allStatic.hpp"
 #include "oops/oopsHierarchy.hpp"
 
-class ZOop : public AllStatic {
+class XOop : public AllStatic {
 public:
   static oop from_address(uintptr_t addr);
   static uintptr_t to_address(oop o);
 };
 
-#endif // SHARE_GC_Z_ZOOP_HPP
+#endif // SHARE_GC_X_XOOP_HPP
diff --git a/src/hotspot/share/gc/z/zOop.inline.hpp b/src/hotspot/share/gc/x/xOop.inline.hpp
similarity index 83%
rename from src/hotspot/share/gc/z/zOop.inline.hpp
rename to src/hotspot/share/gc/x/xOop.inline.hpp
index e71e233ffc9..933987577d1 100644
--- a/src/hotspot/share/gc/z/zOop.inline.hpp
+++ b/src/hotspot/share/gc/x/xOop.inline.hpp
@@ -21,17 +21,17 @@
  * questions.
  */
 
-#ifndef SHARE_GC_Z_ZOOP_INLINE_HPP
-#define SHARE_GC_Z_ZOOP_INLINE_HPP
+#ifndef SHARE_GC_X_XOOP_INLINE_HPP
+#define SHARE_GC_X_XOOP_INLINE_HPP
 
-#include "gc/z/zOop.hpp"
+#include "gc/x/xOop.hpp"
 
-inline oop ZOop::from_address(uintptr_t addr) {
+inline oop XOop::from_address(uintptr_t addr) {
   return cast_to_oop(addr);
 }
 
-inline uintptr_t ZOop::to_address(oop o) {
+inline uintptr_t XOop::to_address(oop o) {
   return cast_from_oop<uintptr_t>(o);
 }
 
-#endif // SHARE_GC_Z_ZOOP_INLINE_HPP
+#endif // SHARE_GC_X_XOOP_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xPage.cpp b/src/hotspot/share/gc/x/xPage.cpp
new file mode 100644
index 00000000000..896adb82768
--- /dev/null
+++ b/src/hotspot/share/gc/x/xPage.cpp
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xList.inline.hpp"
+#include "gc/x/xPage.inline.hpp"
+#include "gc/x/xPhysicalMemory.inline.hpp"
+#include "gc/x/xVirtualMemory.inline.hpp"
+#include "utilities/align.hpp"
+#include "utilities/debug.hpp"
+
+XPage::XPage(const XVirtualMemory& vmem, const XPhysicalMemory& pmem) :
+    XPage(type_from_size(vmem.size()), vmem, pmem) {}
+
+XPage::XPage(uint8_t type, const XVirtualMemory& vmem, const XPhysicalMemory& pmem) :
+    _type(type),
+    _numa_id((uint8_t)-1),
+    _seqnum(0),
+    _virtual(vmem),
+    _top(start()),
+    _livemap(object_max_count()),
+    _last_used(0),
+    _physical(pmem),
+    _node() {
+  assert_initialized();
+}
+
+XPage::~XPage() {}
+
+void XPage::assert_initialized() const {
+  assert(!_virtual.is_null(), "Should not be null");
+  assert(!_physical.is_null(), "Should not be null");
+  assert(_virtual.size() == _physical.size(), "Virtual/Physical size mismatch");
+  assert((_type == XPageTypeSmall && size() == XPageSizeSmall) ||
+         (_type == XPageTypeMedium && size() == XPageSizeMedium) ||
+         (_type == XPageTypeLarge && is_aligned(size(), XGranuleSize)),
+         "Page type/size mismatch");
+}
+
+void XPage::reset() {
+  _seqnum = XGlobalSeqNum;
+  _top = start();
+  _livemap.reset();
+  _last_used = 0;
+}
+
+void XPage::reset_for_in_place_relocation() {
+  _seqnum = XGlobalSeqNum;
+  _top = start();
+}
+
+XPage* XPage::retype(uint8_t type) {
+  assert(_type != type, "Invalid retype");
+  _type = type;
+  _livemap.resize(object_max_count());
+  return this;
+}
+
+XPage* XPage::split(size_t size) {
+  return split(type_from_size(size), size);
+}
+
+XPage* XPage::split(uint8_t type, size_t size) {
+  assert(_virtual.size() > size, "Invalid split");
+
+  // Resize this page, keep _numa_id, _seqnum, and _last_used
+  const XVirtualMemory vmem = _virtual.split(size);
+  const XPhysicalMemory pmem = _physical.split(size);
+  _type = type_from_size(_virtual.size());
+  _top = start();
+  _livemap.resize(object_max_count());
+
+  // Create new page, inherit _seqnum and _last_used
+  XPage* const page = new XPage(type, vmem, pmem);
+  page->_seqnum = _seqnum;
+  page->_last_used = _last_used;
+  return page;
+}
+
+XPage* XPage::split_committed() {
+  // Split any committed part of this page into a separate page,
+  // leaving this page with only uncommitted physical memory.
+  const XPhysicalMemory pmem = _physical.split_committed();
+  if (pmem.is_null()) {
+    // Nothing committed
+    return NULL;
+  }
+
+  assert(!_physical.is_null(), "Should not be null");
+
+  // Resize this page
+  const XVirtualMemory vmem = _virtual.split(pmem.size());
+  _type = type_from_size(_virtual.size());
+  _top = start();
+  _livemap.resize(object_max_count());
+
+  // Create new page
+  return new XPage(vmem, pmem);
+}
+
+void XPage::print_on(outputStream* out) const {
+  out->print_cr(" %-6s  " PTR_FORMAT " " PTR_FORMAT " " PTR_FORMAT " %s%s",
+                type_to_string(), start(), top(), end(),
+                is_allocating()  ? " Allocating"  : "",
+                is_relocatable() ? " Relocatable" : "");
+}
+
+void XPage::print() const {
+  print_on(tty);
+}
+
+void XPage::verify_live(uint32_t live_objects, size_t live_bytes) const {
+  guarantee(live_objects == _livemap.live_objects(), "Invalid number of live objects");
+  guarantee(live_bytes == _livemap.live_bytes(), "Invalid number of live bytes");
+}
diff --git a/src/hotspot/share/gc/x/xPage.hpp b/src/hotspot/share/gc/x/xPage.hpp
new file mode 100644
index 00000000000..c1040e034bd
--- /dev/null
+++ b/src/hotspot/share/gc/x/xPage.hpp
@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XPAGE_HPP
+#define SHARE_GC_X_XPAGE_HPP
+
+#include "gc/x/xList.hpp"
+#include "gc/x/xLiveMap.hpp"
+#include "gc/x/xPhysicalMemory.hpp"
+#include "gc/x/xVirtualMemory.hpp"
+#include "memory/allocation.hpp"
+
+class VMStructs;
+
+class XPage : public CHeapObj<mtGC> {
+  friend class ::VMStructs;
+  friend class XList<XPage>;
+
+private:
+  uint8_t            _type;
+  uint8_t            _numa_id;
+  uint32_t           _seqnum;
+  XVirtualMemory     _virtual;
+  volatile uintptr_t _top;
+  XLiveMap           _livemap;
+  uint64_t           _last_used;
+  XPhysicalMemory    _physical;
+  XListNode<XPage>   _node;
+
+  void assert_initialized() const;
+
+  uint8_t type_from_size(size_t size) const;
+  const char* type_to_string() const;
+
+  bool is_object_marked(uintptr_t addr) const;
+  bool is_object_strongly_marked(uintptr_t addr) const;
+
+public:
+  XPage(const XVirtualMemory& vmem, const XPhysicalMemory& pmem);
+  XPage(uint8_t type, const XVirtualMemory& vmem, const XPhysicalMemory& pmem);
+  ~XPage();
+
+  uint32_t object_max_count() const;
+  size_t object_alignment_shift() const;
+  size_t object_alignment() const;
+
+  uint8_t type() const;
+  uintptr_t start() const;
+  uintptr_t end() const;
+  size_t size() const;
+  uintptr_t top() const;
+  size_t remaining() const;
+
+  const XVirtualMemory& virtual_memory() const;
+  const XPhysicalMemory& physical_memory() const;
+  XPhysicalMemory& physical_memory();
+
+  uint8_t numa_id();
+
+  bool is_allocating() const;
+  bool is_relocatable() const;
+
+  uint64_t last_used() const;
+  void set_last_used();
+
+  void reset();
+  void reset_for_in_place_relocation();
+
+  XPage* retype(uint8_t type);
+  XPage* split(size_t size);
+  XPage* split(uint8_t type, size_t size);
+  XPage* split_committed();
+
+  bool is_in(uintptr_t addr) const;
+
+  bool is_marked() const;
+  template <bool finalizable> bool is_object_marked(uintptr_t addr) const;
+  bool is_object_live(uintptr_t addr) const;
+  bool is_object_strongly_live(uintptr_t addr) const;
+  bool mark_object(uintptr_t addr, bool finalizable, bool& inc_live);
+
+  void inc_live(uint32_t objects, size_t bytes);
+  uint32_t live_objects() const;
+  size_t live_bytes() const;
+
+  void object_iterate(ObjectClosure* cl);
+
+  uintptr_t alloc_object(size_t size);
+  uintptr_t alloc_object_atomic(size_t size);
+
+  bool undo_alloc_object(uintptr_t addr, size_t size);
+  bool undo_alloc_object_atomic(uintptr_t addr, size_t size);
+
+  void print_on(outputStream* out) const;
+  void print() const;
+
+  void verify_live(uint32_t live_objects, size_t live_bytes) const;
+};
+
+class XPageClosure {
+public:
+  virtual void do_page(const XPage* page) = 0;
+};
+
+#endif // SHARE_GC_X_XPAGE_HPP
diff --git a/src/hotspot/share/gc/x/xPage.inline.hpp b/src/hotspot/share/gc/x/xPage.inline.hpp
new file mode 100644
index 00000000000..8017b2dadf6
--- /dev/null
+++ b/src/hotspot/share/gc/x/xPage.inline.hpp
@@ -0,0 +1,312 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XPAGE_INLINE_HPP
+#define SHARE_GC_X_XPAGE_INLINE_HPP
+
+#include "gc/x/xPage.hpp"
+
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xLiveMap.inline.hpp"
+#include "gc/x/xNUMA.hpp"
+#include "gc/x/xPhysicalMemory.inline.hpp"
+#include "gc/x/xVirtualMemory.inline.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/os.hpp"
+#include "utilities/align.hpp"
+#include "utilities/debug.hpp"
+
+inline uint8_t XPage::type_from_size(size_t size) const {
+  if (size == XPageSizeSmall) {
+    return XPageTypeSmall;
+  } else if (size == XPageSizeMedium) {
+    return XPageTypeMedium;
+  } else {
+    return XPageTypeLarge;
+  }
+}
+
+inline const char* XPage::type_to_string() const {
+  switch (type()) {
+  case XPageTypeSmall:
+    return "Small";
+
+  case XPageTypeMedium:
+    return "Medium";
+
+  default:
+    assert(type() == XPageTypeLarge, "Invalid page type");
+    return "Large";
+  }
+}
+
+inline uint32_t XPage::object_max_count() const {
+  switch (type()) {
+  case XPageTypeLarge:
+    // A large page can only contain a single
+    // object aligned to the start of the page.
+    return 1;
+
+  default:
+    return (uint32_t)(size() >> object_alignment_shift());
+  }
+}
+
+inline size_t XPage::object_alignment_shift() const {
+  switch (type()) {
+  case XPageTypeSmall:
+    return XObjectAlignmentSmallShift;
+
+  case XPageTypeMedium:
+    return XObjectAlignmentMediumShift;
+
+  default:
+    assert(type() == XPageTypeLarge, "Invalid page type");
+    return XObjectAlignmentLargeShift;
+  }
+}
+
+inline size_t XPage::object_alignment() const {
+  switch (type()) {
+  case XPageTypeSmall:
+    return XObjectAlignmentSmall;
+
+  case XPageTypeMedium:
+    return XObjectAlignmentMedium;
+
+  default:
+    assert(type() == XPageTypeLarge, "Invalid page type");
+    return XObjectAlignmentLarge;
+  }
+}
+
+inline uint8_t XPage::type() const {
+  return _type;
+}
+
+inline uintptr_t XPage::start() const {
+  return _virtual.start();
+}
+
+inline uintptr_t XPage::end() const {
+  return _virtual.end();
+}
+
+inline size_t XPage::size() const {
+  return _virtual.size();
+}
+
+inline uintptr_t XPage::top() const {
+  return _top;
+}
+
+inline size_t XPage::remaining() const {
+  return end() - top();
+}
+
+inline const XVirtualMemory& XPage::virtual_memory() const {
+  return _virtual;
+}
+
+inline const XPhysicalMemory& XPage::physical_memory() const {
+  return _physical;
+}
+
+inline XPhysicalMemory& XPage::physical_memory() {
+  return _physical;
+}
+
+inline uint8_t XPage::numa_id() {
+  if (_numa_id == (uint8_t)-1) {
+    _numa_id = XNUMA::memory_id(XAddress::good(start()));
+  }
+
+  return _numa_id;
+}
+
+inline bool XPage::is_allocating() const {
+  return _seqnum == XGlobalSeqNum;
+}
+
+inline bool XPage::is_relocatable() const {
+  return _seqnum < XGlobalSeqNum;
+}
+
+inline uint64_t XPage::last_used() const {
+  return _last_used;
+}
+
+inline void XPage::set_last_used() {
+  _last_used = ceil(os::elapsedTime());
+}
+
+inline bool XPage::is_in(uintptr_t addr) const {
+  const uintptr_t offset = XAddress::offset(addr);
+  return offset >= start() && offset < top();
+}
+
+inline bool XPage::is_marked() const {
+  assert(is_relocatable(), "Invalid page state");
+  return _livemap.is_marked();
+}
+
+inline bool XPage::is_object_marked(uintptr_t addr) const {
+  assert(is_relocatable(), "Invalid page state");
+  const size_t index = ((XAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
+  return _livemap.get(index);
+}
+
+inline bool XPage::is_object_strongly_marked(uintptr_t addr) const {
+  assert(is_relocatable(), "Invalid page state");
+  const size_t index = ((XAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
+  return _livemap.get(index + 1);
+}
+
+template <bool finalizable>
+inline bool XPage::is_object_marked(uintptr_t addr) const {
+  return finalizable ? is_object_marked(addr) : is_object_strongly_marked(addr);
+}
+
+inline bool XPage::is_object_live(uintptr_t addr) const {
+  return is_allocating() || is_object_marked(addr);
+}
+
+inline bool XPage::is_object_strongly_live(uintptr_t addr) const {
+  return is_allocating() || is_object_strongly_marked(addr);
+}
+
+inline bool XPage::mark_object(uintptr_t addr, bool finalizable, bool& inc_live) {
+  assert(XAddress::is_marked(addr), "Invalid address");
+  assert(is_relocatable(), "Invalid page state");
+  assert(is_in(addr), "Invalid address");
+
+  // Set mark bit
+  const size_t index = ((XAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
+  return _livemap.set(index, finalizable, inc_live);
+}
+
+inline void XPage::inc_live(uint32_t objects, size_t bytes) {
+  _livemap.inc_live(objects, bytes);
+}
+
+inline uint32_t XPage::live_objects() const {
+  assert(is_marked(), "Should be marked");
+  return _livemap.live_objects();
+}
+
+inline size_t XPage::live_bytes() const {
+  assert(is_marked(), "Should be marked");
+  return _livemap.live_bytes();
+}
+
+inline void XPage::object_iterate(ObjectClosure* cl) {
+  _livemap.iterate(cl, XAddress::good(start()), object_alignment_shift());
+}
+
+inline uintptr_t XPage::alloc_object(size_t size) {
+  assert(is_allocating(), "Invalid state");
+
+  const size_t aligned_size = align_up(size, object_alignment());
+  const uintptr_t addr = top();
+  const uintptr_t new_top = addr + aligned_size;
+
+  if (new_top > end()) {
+    // Not enough space left
+    return 0;
+  }
+
+  _top = new_top;
+
+  return XAddress::good(addr);
+}
+
+inline uintptr_t XPage::alloc_object_atomic(size_t size) {
+  assert(is_allocating(), "Invalid state");
+
+  const size_t aligned_size = align_up(size, object_alignment());
+  uintptr_t addr = top();
+
+  for (;;) {
+    const uintptr_t new_top = addr + aligned_size;
+    if (new_top > end()) {
+      // Not enough space left
+      return 0;
+    }
+
+    const uintptr_t prev_top = Atomic::cmpxchg(&_top, addr, new_top);
+    if (prev_top == addr) {
+      // Success
+      return XAddress::good(addr);
+    }
+
+    // Retry
+    addr = prev_top;
+  }
+}
+
+inline bool XPage::undo_alloc_object(uintptr_t addr, size_t size) {
+  assert(is_allocating(), "Invalid state");
+
+  const uintptr_t offset = XAddress::offset(addr);
+  const size_t aligned_size = align_up(size, object_alignment());
+  const uintptr_t old_top = top();
+  const uintptr_t new_top = old_top - aligned_size;
+
+  if (new_top != offset) {
+    // Failed to undo allocation, not the last allocated object
+    return false;
+  }
+
+  _top = new_top;
+
+  // Success
+  return true;
+}
+
+inline bool XPage::undo_alloc_object_atomic(uintptr_t addr, size_t size) {
+  assert(is_allocating(), "Invalid state");
+
+  const uintptr_t offset = XAddress::offset(addr);
+  const size_t aligned_size = align_up(size, object_alignment());
+  uintptr_t old_top = top();
+
+  for (;;) {
+    const uintptr_t new_top = old_top - aligned_size;
+    if (new_top != offset) {
+      // Failed to undo allocation, not the last allocated object
+      return false;
+    }
+
+    const uintptr_t prev_top = Atomic::cmpxchg(&_top, old_top, new_top);
+    if (prev_top == old_top) {
+      // Success
+      return true;
+    }
+
+    // Retry
+    old_top = prev_top;
+  }
+}
+
+#endif // SHARE_GC_X_XPAGE_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xPageAllocator.cpp b/src/hotspot/share/gc/x/xPageAllocator.cpp
new file mode 100644
index 00000000000..49b7989c708
--- /dev/null
+++ b/src/hotspot/share/gc/x/xPageAllocator.cpp
@@ -0,0 +1,870 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/shared/suspendibleThreadSet.hpp"
+#include "gc/x/xArray.inline.hpp"
+#include "gc/x/xCollectedHeap.hpp"
+#include "gc/x/xFuture.inline.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xLock.inline.hpp"
+#include "gc/x/xPage.inline.hpp"
+#include "gc/x/xPageAllocator.inline.hpp"
+#include "gc/x/xPageCache.hpp"
+#include "gc/x/xSafeDelete.inline.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xTask.hpp"
+#include "gc/x/xUncommitter.hpp"
+#include "gc/x/xUnmapper.hpp"
+#include "gc/x/xWorkers.hpp"
+#include "jfr/jfrEvents.hpp"
+#include "logging/log.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/init.hpp"
+#include "runtime/java.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+static const XStatCounter       XCounterAllocationRate("Memory", "Allocation Rate", XStatUnitBytesPerSecond);
+static const XStatCounter       XCounterPageCacheFlush("Memory", "Page Cache Flush", XStatUnitBytesPerSecond);
+static const XStatCounter       XCounterDefragment("Memory", "Defragment", XStatUnitOpsPerSecond);
+static const XStatCriticalPhase XCriticalPhaseAllocationStall("Allocation Stall");
+
+enum XPageAllocationStall {
+  XPageAllocationStallSuccess,
+  XPageAllocationStallFailed,
+  XPageAllocationStallStartGC
+};
+
+class XPageAllocation : public StackObj {
+  friend class XList<XPageAllocation>;
+
+private:
+  const uint8_t                 _type;
+  const size_t                  _size;
+  const XAllocationFlags        _flags;
+  const uint32_t                _seqnum;
+  size_t                        _flushed;
+  size_t                        _committed;
+  XList<XPage>                  _pages;
+  XListNode<XPageAllocation>    _node;
+  XFuture<XPageAllocationStall> _stall_result;
+
+public:
+  XPageAllocation(uint8_t type, size_t size, XAllocationFlags flags) :
+      _type(type),
+      _size(size),
+      _flags(flags),
+      _seqnum(XGlobalSeqNum),
+      _flushed(0),
+      _committed(0),
+      _pages(),
+      _node(),
+      _stall_result() {}
+
+  uint8_t type() const {
+    return _type;
+  }
+
+  size_t size() const {
+    return _size;
+  }
+
+  XAllocationFlags flags() const {
+    return _flags;
+  }
+
+  uint32_t seqnum() const {
+    return _seqnum;
+  }
+
+  size_t flushed() const {
+    return _flushed;
+  }
+
+  void set_flushed(size_t flushed) {
+    _flushed = flushed;
+  }
+
+  size_t committed() const {
+    return _committed;
+  }
+
+  void set_committed(size_t committed) {
+    _committed = committed;
+  }
+
+  XPageAllocationStall wait() {
+    return _stall_result.get();
+  }
+
+  XList<XPage>* pages() {
+    return &_pages;
+  }
+
+  void satisfy(XPageAllocationStall result) {
+    _stall_result.set(result);
+  }
+};
+
+XPageAllocator::XPageAllocator(XWorkers* workers,
+                               size_t min_capacity,
+                               size_t initial_capacity,
+                               size_t max_capacity) :
+    _lock(),
+    _cache(),
+    _virtual(max_capacity),
+    _physical(max_capacity),
+    _min_capacity(min_capacity),
+    _max_capacity(max_capacity),
+    _current_max_capacity(max_capacity),
+    _capacity(0),
+    _claimed(0),
+    _used(0),
+    _used_high(0),
+    _used_low(0),
+    _reclaimed(0),
+    _stalled(),
+    _nstalled(0),
+    _satisfied(),
+    _unmapper(new XUnmapper(this)),
+    _uncommitter(new XUncommitter(this)),
+    _safe_delete(),
+    _initialized(false) {
+
+  if (!_virtual.is_initialized() || !_physical.is_initialized()) {
+    return;
+  }
+
+  log_info_p(gc, init)("Min Capacity: " SIZE_FORMAT "M", min_capacity / M);
+  log_info_p(gc, init)("Initial Capacity: " SIZE_FORMAT "M", initial_capacity / M);
+  log_info_p(gc, init)("Max Capacity: " SIZE_FORMAT "M", max_capacity / M);
+  if (XPageSizeMedium > 0) {
+    log_info_p(gc, init)("Medium Page Size: " SIZE_FORMAT "M", XPageSizeMedium / M);
+  } else {
+    log_info_p(gc, init)("Medium Page Size: N/A");
+  }
+  log_info_p(gc, init)("Pre-touch: %s", AlwaysPreTouch ? "Enabled" : "Disabled");
+
+  // Warn if system limits could stop us from reaching max capacity
+  _physical.warn_commit_limits(max_capacity);
+
+  // Check if uncommit should and can be enabled
+  _physical.try_enable_uncommit(min_capacity, max_capacity);
+
+  // Pre-map initial capacity
+  if (!prime_cache(workers, initial_capacity)) {
+    log_error_p(gc)("Failed to allocate initial Java heap (" SIZE_FORMAT "M)", initial_capacity / M);
+    return;
+  }
+
+  // Successfully initialized
+  _initialized = true;
+}
+
+class XPreTouchTask : public XTask {
+private:
+  const XPhysicalMemoryManager* const _physical;
+  volatile uintptr_t                  _start;
+  const uintptr_t                     _end;
+
+public:
+  XPreTouchTask(const XPhysicalMemoryManager* physical, uintptr_t start, uintptr_t end) :
+      XTask("XPreTouchTask"),
+      _physical(physical),
+      _start(start),
+      _end(end) {}
+
+  virtual void work() {
+    for (;;) {
+      // Get granule offset
+      const size_t size = XGranuleSize;
+      const uintptr_t offset = Atomic::fetch_and_add(&_start, size);
+      if (offset >= _end) {
+        // Done
+        break;
+      }
+
+      // Pre-touch granule
+      _physical->pretouch(offset, size);
+    }
+  }
+};
+
+bool XPageAllocator::prime_cache(XWorkers* workers, size_t size) {
+  XAllocationFlags flags;
+
+  flags.set_non_blocking();
+  flags.set_low_address();
+
+  XPage* const page = alloc_page(XPageTypeLarge, size, flags);
+  if (page == NULL) {
+    return false;
+  }
+
+  if (AlwaysPreTouch) {
+    // Pre-touch page
+    XPreTouchTask task(&_physical, page->start(), page->end());
+    workers->run_all(&task);
+  }
+
+  free_page(page, false /* reclaimed */);
+
+  return true;
+}
+
+bool XPageAllocator::is_initialized() const {
+  return _initialized;
+}
+
+size_t XPageAllocator::min_capacity() const {
+  return _min_capacity;
+}
+
+size_t XPageAllocator::max_capacity() const {
+  return _max_capacity;
+}
+
+size_t XPageAllocator::soft_max_capacity() const {
+  // Note that SoftMaxHeapSize is a manageable flag
+  const size_t soft_max_capacity = Atomic::load(&SoftMaxHeapSize);
+  const size_t current_max_capacity = Atomic::load(&_current_max_capacity);
+  return MIN2(soft_max_capacity, current_max_capacity);
+}
+
+size_t XPageAllocator::capacity() const {
+  return Atomic::load(&_capacity);
+}
+
+size_t XPageAllocator::used() const {
+  return Atomic::load(&_used);
+}
+
+size_t XPageAllocator::unused() const {
+  const ssize_t capacity = (ssize_t)Atomic::load(&_capacity);
+  const ssize_t used = (ssize_t)Atomic::load(&_used);
+  const ssize_t claimed = (ssize_t)Atomic::load(&_claimed);
+  const ssize_t unused = capacity - used - claimed;
+  return unused > 0 ? (size_t)unused : 0;
+}
+
+XPageAllocatorStats XPageAllocator::stats() const {
+  XLocker<XLock> locker(&_lock);
+  return XPageAllocatorStats(_min_capacity,
+                             _max_capacity,
+                             soft_max_capacity(),
+                             _capacity,
+                             _used,
+                             _used_high,
+                             _used_low,
+                             _reclaimed);
+}
+
+void XPageAllocator::reset_statistics() {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+  _reclaimed = 0;
+  _used_high = _used_low = _used;
+  _nstalled = 0;
+}
+
+size_t XPageAllocator::increase_capacity(size_t size) {
+  const size_t increased = MIN2(size, _current_max_capacity - _capacity);
+
+  if (increased > 0) {
+    // Update atomically since we have concurrent readers
+    Atomic::add(&_capacity, increased);
+
+    // Record time of last commit. When allocation, we prefer increasing
+    // the capacity over flushing the cache. That means there could be
+    // expired pages in the cache at this time. However, since we are
+    // increasing the capacity we are obviously in need of committed
+    // memory and should therefore not be uncommitting memory.
+    _cache.set_last_commit();
+  }
+
+  return increased;
+}
+
+void XPageAllocator::decrease_capacity(size_t size, bool set_max_capacity) {
+  // Update atomically since we have concurrent readers
+  Atomic::sub(&_capacity, size);
+
+  if (set_max_capacity) {
+    // Adjust current max capacity to avoid further attempts to increase capacity
+    log_error_p(gc)("Forced to lower max Java heap size from "
+                    SIZE_FORMAT "M(%.0f%%) to " SIZE_FORMAT "M(%.0f%%)",
+                    _current_max_capacity / M, percent_of(_current_max_capacity, _max_capacity),
+                    _capacity / M, percent_of(_capacity, _max_capacity));
+
+    // Update atomically since we have concurrent readers
+    Atomic::store(&_current_max_capacity, _capacity);
+  }
+}
+
+void XPageAllocator::increase_used(size_t size, bool worker_relocation) {
+  if (worker_relocation) {
+    // Allocating a page for the purpose of worker relocation has
+    // a negative contribution to the number of reclaimed bytes.
+    _reclaimed -= size;
+  }
+
+  // Update atomically since we have concurrent readers
+  const size_t used = Atomic::add(&_used, size);
+  if (used > _used_high) {
+    _used_high = used;
+  }
+}
+
+void XPageAllocator::decrease_used(size_t size, bool reclaimed) {
+  // Only pages explicitly released with the reclaimed flag set
+  // counts as reclaimed bytes. This flag is true when we release
+  // a page after relocation, and is false when we release a page
+  // to undo an allocation.
+  if (reclaimed) {
+    _reclaimed += size;
+  }
+
+  // Update atomically since we have concurrent readers
+  const size_t used = Atomic::sub(&_used, size);
+  if (used < _used_low) {
+    _used_low = used;
+  }
+}
+
+bool XPageAllocator::commit_page(XPage* page) {
+  // Commit physical memory
+  return _physical.commit(page->physical_memory());
+}
+
+void XPageAllocator::uncommit_page(XPage* page) {
+  if (!ZUncommit) {
+    return;
+  }
+
+  // Uncommit physical memory
+  _physical.uncommit(page->physical_memory());
+}
+
+void XPageAllocator::map_page(const XPage* page) const {
+  // Map physical memory
+  _physical.map(page->start(), page->physical_memory());
+}
+
+void XPageAllocator::unmap_page(const XPage* page) const {
+  // Unmap physical memory
+  _physical.unmap(page->start(), page->size());
+}
+
+void XPageAllocator::destroy_page(XPage* page) {
+  // Free virtual memory
+  _virtual.free(page->virtual_memory());
+
+  // Free physical memory
+  _physical.free(page->physical_memory());
+
+  // Delete page safely
+  _safe_delete(page);
+}
+
+bool XPageAllocator::is_alloc_allowed(size_t size) const {
+  const size_t available = _current_max_capacity - _used - _claimed;
+  return available >= size;
+}
+
+bool XPageAllocator::alloc_page_common_inner(uint8_t type, size_t size, XList<XPage>* pages) {
+  if (!is_alloc_allowed(size)) {
+    // Out of memory
+    return false;
+  }
+
+  // Try allocate from the page cache
+  XPage* const page = _cache.alloc_page(type, size);
+  if (page != NULL) {
+    // Success
+    pages->insert_last(page);
+    return true;
+  }
+
+  // Try increase capacity
+  const size_t increased = increase_capacity(size);
+  if (increased < size) {
+    // Could not increase capacity enough to satisfy the allocation
+    // completely. Flush the page cache to satisfy the remainder.
+    const size_t remaining = size - increased;
+    _cache.flush_for_allocation(remaining, pages);
+  }
+
+  // Success
+  return true;
+}
+
+bool XPageAllocator::alloc_page_common(XPageAllocation* allocation) {
+  const uint8_t type = allocation->type();
+  const size_t size = allocation->size();
+  const XAllocationFlags flags = allocation->flags();
+  XList<XPage>* const pages = allocation->pages();
+
+  if (!alloc_page_common_inner(type, size, pages)) {
+    // Out of memory
+    return false;
+  }
+
+  // Updated used statistics
+  increase_used(size, flags.worker_relocation());
+
+  // Success
+  return true;
+}
+
+static void check_out_of_memory_during_initialization() {
+  if (!is_init_completed()) {
+    vm_exit_during_initialization("java.lang.OutOfMemoryError", "Java heap too small");
+  }
+}
+
+bool XPageAllocator::alloc_page_stall(XPageAllocation* allocation) {
+  XStatTimer timer(XCriticalPhaseAllocationStall);
+  EventZAllocationStall event;
+  XPageAllocationStall result;
+
+  // We can only block if the VM is fully initialized
+  check_out_of_memory_during_initialization();
+
+  // Increment stalled counter
+  Atomic::inc(&_nstalled);
+
+  do {
+    // Start asynchronous GC
+    XCollectedHeap::heap()->collect(GCCause::_z_allocation_stall);
+
+    // Wait for allocation to complete, fail or request a GC
+    result = allocation->wait();
+  } while (result == XPageAllocationStallStartGC);
+
+  {
+    //
+    // We grab the lock here for two different reasons:
+    //
+    // 1) Guard deletion of underlying semaphore. This is a workaround for
+    // a bug in sem_post() in glibc < 2.21, where it's not safe to destroy
+    // the semaphore immediately after returning from sem_wait(). The
+    // reason is that sem_post() can touch the semaphore after a waiting
+    // thread have returned from sem_wait(). To avoid this race we are
+    // forcing the waiting thread to acquire/release the lock held by the
+    // posting thread. https://sourceware.org/bugzilla/show_bug.cgi?id=12674
+    //
+    // 2) Guard the list of satisfied pages.
+    //
+    XLocker<XLock> locker(&_lock);
+    _satisfied.remove(allocation);
+  }
+
+  // Send event
+  event.commit(allocation->type(), allocation->size());
+
+  return (result == XPageAllocationStallSuccess);
+}
+
+bool XPageAllocator::alloc_page_or_stall(XPageAllocation* allocation) {
+  {
+    XLocker<XLock> locker(&_lock);
+
+    if (alloc_page_common(allocation)) {
+      // Success
+      return true;
+    }
+
+    // Failed
+    if (allocation->flags().non_blocking()) {
+      // Don't stall
+      return false;
+    }
+
+    // Enqueue allocation request
+    _stalled.insert_last(allocation);
+  }
+
+  // Stall
+  return alloc_page_stall(allocation);
+}
+
+XPage* XPageAllocator::alloc_page_create(XPageAllocation* allocation) {
+  const size_t size = allocation->size();
+
+  // Allocate virtual memory. To make error handling a lot more straight
+  // forward, we allocate virtual memory before destroying flushed pages.
+  // Flushed pages are also unmapped and destroyed asynchronously, so we
+  // can't immediately reuse that part of the address space anyway.
+  const XVirtualMemory vmem = _virtual.alloc(size, allocation->flags().low_address());
+  if (vmem.is_null()) {
+    log_error(gc)("Out of address space");
+    return NULL;
+  }
+
+  XPhysicalMemory pmem;
+  size_t flushed = 0;
+
+  // Harvest physical memory from flushed pages
+  XListRemoveIterator<XPage> iter(allocation->pages());
+  for (XPage* page; iter.next(&page);) {
+    flushed += page->size();
+
+    // Harvest flushed physical memory
+    XPhysicalMemory& fmem = page->physical_memory();
+    pmem.add_segments(fmem);
+    fmem.remove_segments();
+
+    // Unmap and destroy page
+    _unmapper->unmap_and_destroy_page(page);
+  }
+
+  if (flushed > 0) {
+    allocation->set_flushed(flushed);
+
+    // Update statistics
+    XStatInc(XCounterPageCacheFlush, flushed);
+    log_debug(gc, heap)("Page Cache Flushed: " SIZE_FORMAT "M", flushed / M);
+  }
+
+  // Allocate any remaining physical memory. Capacity and used has
+  // already been adjusted, we just need to fetch the memory, which
+  // is guaranteed to succeed.
+  if (flushed < size) {
+    const size_t remaining = size - flushed;
+    allocation->set_committed(remaining);
+    _physical.alloc(pmem, remaining);
+  }
+
+  // Create new page
+  return new XPage(allocation->type(), vmem, pmem);
+}
+
+bool XPageAllocator::should_defragment(const XPage* page) const {
+  // A small page can end up at a high address (second half of the address space)
+  // if we've split a larger page or we have a constrained address space. To help
+  // fight address space fragmentation we remap such pages to a lower address, if
+  // a lower address is available.
+  return page->type() == XPageTypeSmall &&
+         page->start() >= _virtual.reserved() / 2 &&
+         page->start() > _virtual.lowest_available_address();
+}
+
+bool XPageAllocator::is_alloc_satisfied(XPageAllocation* allocation) const {
+  // The allocation is immediately satisfied if the list of pages contains
+  // exactly one page, with the type and size that was requested. However,
+  // even if the allocation is immediately satisfied we might still want to
+  // return false here to force the page to be remapped to fight address
+  // space fragmentation.
+
+  if (allocation->pages()->size() != 1) {
+    // Not a single page
+    return false;
+  }
+
+  const XPage* const page = allocation->pages()->first();
+  if (page->type() != allocation->type() ||
+      page->size() != allocation->size()) {
+    // Wrong type or size
+    return false;
+  }
+
+  if (should_defragment(page)) {
+    // Defragment address space
+    XStatInc(XCounterDefragment);
+    return false;
+  }
+
+  // Allocation immediately satisfied
+  return true;
+}
+
+XPage* XPageAllocator::alloc_page_finalize(XPageAllocation* allocation) {
+  // Fast path
+  if (is_alloc_satisfied(allocation)) {
+    return allocation->pages()->remove_first();
+  }
+
+  // Slow path
+  XPage* const page = alloc_page_create(allocation);
+  if (page == NULL) {
+    // Out of address space
+    return NULL;
+  }
+
+  // Commit page
+  if (commit_page(page)) {
+    // Success
+    map_page(page);
+    return page;
+  }
+
+  // Failed or partially failed. Split of any successfully committed
+  // part of the page into a new page and insert it into list of pages,
+  // so that it will be re-inserted into the page cache.
+  XPage* const committed_page = page->split_committed();
+  destroy_page(page);
+
+  if (committed_page != NULL) {
+    map_page(committed_page);
+    allocation->pages()->insert_last(committed_page);
+  }
+
+  return NULL;
+}
+
+void XPageAllocator::alloc_page_failed(XPageAllocation* allocation) {
+  XLocker<XLock> locker(&_lock);
+
+  size_t freed = 0;
+
+  // Free any allocated/flushed pages
+  XListRemoveIterator<XPage> iter(allocation->pages());
+  for (XPage* page; iter.next(&page);) {
+    freed += page->size();
+    free_page_inner(page, false /* reclaimed */);
+  }
+
+  // Adjust capacity and used to reflect the failed capacity increase
+  const size_t remaining = allocation->size() - freed;
+  decrease_used(remaining, false /* reclaimed */);
+  decrease_capacity(remaining, true /* set_max_capacity */);
+
+  // Try satisfy stalled allocations
+  satisfy_stalled();
+}
+
+XPage* XPageAllocator::alloc_page(uint8_t type, size_t size, XAllocationFlags flags) {
+  EventZPageAllocation event;
+
+retry:
+  XPageAllocation allocation(type, size, flags);
+
+  // Allocate one or more pages from the page cache. If the allocation
+  // succeeds but the returned pages don't cover the complete allocation,
+  // then finalize phase is allowed to allocate the remaining memory
+  // directly from the physical memory manager. Note that this call might
+  // block in a safepoint if the non-blocking flag is not set.
+  if (!alloc_page_or_stall(&allocation)) {
+    // Out of memory
+    return NULL;
+  }
+
+  XPage* const page = alloc_page_finalize(&allocation);
+  if (page == NULL) {
+    // Failed to commit or map. Clean up and retry, in the hope that
+    // we can still allocate by flushing the page cache (more aggressively).
+    alloc_page_failed(&allocation);
+    goto retry;
+  }
+
+  // Reset page. This updates the page's sequence number and must
+  // be done after we potentially blocked in a safepoint (stalled)
+  // where the global sequence number was updated.
+  page->reset();
+
+  // Update allocation statistics. Exclude worker relocations to avoid
+  // artificial inflation of the allocation rate during relocation.
+  if (!flags.worker_relocation() && is_init_completed()) {
+    // Note that there are two allocation rate counters, which have
+    // different purposes and are sampled at different frequencies.
+    const size_t bytes = page->size();
+    XStatInc(XCounterAllocationRate, bytes);
+    XStatInc(XStatAllocRate::counter(), bytes);
+  }
+
+  // Send event
+  event.commit(type, size, allocation.flushed(), allocation.committed(),
+               page->physical_memory().nsegments(), flags.non_blocking());
+
+  return page;
+}
+
+void XPageAllocator::satisfy_stalled() {
+  for (;;) {
+    XPageAllocation* const allocation = _stalled.first();
+    if (allocation == NULL) {
+      // Allocation queue is empty
+      return;
+    }
+
+    if (!alloc_page_common(allocation)) {
+      // Allocation could not be satisfied, give up
+      return;
+    }
+
+    // Allocation succeeded, dequeue and satisfy allocation request.
+    // Note that we must dequeue the allocation request first, since
+    // it will immediately be deallocated once it has been satisfied.
+    _stalled.remove(allocation);
+    _satisfied.insert_last(allocation);
+    allocation->satisfy(XPageAllocationStallSuccess);
+  }
+}
+
+void XPageAllocator::free_page_inner(XPage* page, bool reclaimed) {
+  // Update used statistics
+  decrease_used(page->size(), reclaimed);
+
+  // Set time when last used
+  page->set_last_used();
+
+  // Cache page
+  _cache.free_page(page);
+}
+
+void XPageAllocator::free_page(XPage* page, bool reclaimed) {
+  XLocker<XLock> locker(&_lock);
+
+  // Free page
+  free_page_inner(page, reclaimed);
+
+  // Try satisfy stalled allocations
+  satisfy_stalled();
+}
+
+void XPageAllocator::free_pages(const XArray<XPage*>* pages, bool reclaimed) {
+  XLocker<XLock> locker(&_lock);
+
+  // Free pages
+  XArrayIterator<XPage*> iter(pages);
+  for (XPage* page; iter.next(&page);) {
+    free_page_inner(page, reclaimed);
+  }
+
+  // Try satisfy stalled allocations
+  satisfy_stalled();
+}
+
+size_t XPageAllocator::uncommit(uint64_t* timeout) {
+  // We need to join the suspendible thread set while manipulating capacity and
+  // used, to make sure GC safepoints will have a consistent view. However, when
+  // ZVerifyViews is enabled we need to join at a broader scope to also make sure
+  // we don't change the address good mask after pages have been flushed, and
+  // thereby made invisible to pages_do(), but before they have been unmapped.
+  SuspendibleThreadSetJoiner joiner(ZVerifyViews);
+  XList<XPage> pages;
+  size_t flushed;
+
+  {
+    SuspendibleThreadSetJoiner joiner(!ZVerifyViews);
+    XLocker<XLock> locker(&_lock);
+
+    // Never uncommit below min capacity. We flush out and uncommit chunks at
+    // a time (~0.8% of the max capacity, but at least one granule and at most
+    // 256M), in case demand for memory increases while we are uncommitting.
+    const size_t retain = MAX2(_used, _min_capacity);
+    const size_t release = _capacity - retain;
+    const size_t limit = MIN2(align_up(_current_max_capacity >> 7, XGranuleSize), 256 * M);
+    const size_t flush = MIN2(release, limit);
+
+    // Flush pages to uncommit
+    flushed = _cache.flush_for_uncommit(flush, &pages, timeout);
+    if (flushed == 0) {
+      // Nothing flushed
+      return 0;
+    }
+
+    // Record flushed pages as claimed
+    Atomic::add(&_claimed, flushed);
+  }
+
+  // Unmap, uncommit, and destroy flushed pages
+  XListRemoveIterator<XPage> iter(&pages);
+  for (XPage* page; iter.next(&page);) {
+    unmap_page(page);
+    uncommit_page(page);
+    destroy_page(page);
+  }
+
+  {
+    SuspendibleThreadSetJoiner joiner(!ZVerifyViews);
+    XLocker<XLock> locker(&_lock);
+
+    // Adjust claimed and capacity to reflect the uncommit
+    Atomic::sub(&_claimed, flushed);
+    decrease_capacity(flushed, false /* set_max_capacity */);
+  }
+
+  return flushed;
+}
+
+void XPageAllocator::enable_deferred_delete() const {
+  _safe_delete.enable_deferred_delete();
+}
+
+void XPageAllocator::disable_deferred_delete() const {
+  _safe_delete.disable_deferred_delete();
+}
+
+void XPageAllocator::debug_map_page(const XPage* page) const {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+  _physical.debug_map(page->start(), page->physical_memory());
+}
+
+void XPageAllocator::debug_unmap_page(const XPage* page) const {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+  _physical.debug_unmap(page->start(), page->size());
+}
+
+void XPageAllocator::pages_do(XPageClosure* cl) const {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+
+  XListIterator<XPageAllocation> iter_satisfied(&_satisfied);
+  for (XPageAllocation* allocation; iter_satisfied.next(&allocation);) {
+    XListIterator<XPage> iter_pages(allocation->pages());
+    for (XPage* page; iter_pages.next(&page);) {
+      cl->do_page(page);
+    }
+  }
+
+  _cache.pages_do(cl);
+}
+
+bool XPageAllocator::has_alloc_stalled() const {
+  return Atomic::load(&_nstalled) != 0;
+}
+
+void XPageAllocator::check_out_of_memory() {
+  XLocker<XLock> locker(&_lock);
+
+  // Fail allocation requests that were enqueued before the
+  // last GC cycle started, otherwise start a new GC cycle.
+  for (XPageAllocation* allocation = _stalled.first(); allocation != NULL; allocation = _stalled.first()) {
+    if (allocation->seqnum() == XGlobalSeqNum) {
+      // Start a new GC cycle, keep allocation requests enqueued
+      allocation->satisfy(XPageAllocationStallStartGC);
+      return;
+    }
+
+    // Out of memory, fail allocation request
+    _stalled.remove(allocation);
+    _satisfied.insert_last(allocation);
+    allocation->satisfy(XPageAllocationStallFailed);
+  }
+}
+
+void XPageAllocator::threads_do(ThreadClosure* tc) const {
+  tc->do_thread(_unmapper);
+  tc->do_thread(_uncommitter);
+}
diff --git a/src/hotspot/share/gc/x/xPageAllocator.hpp b/src/hotspot/share/gc/x/xPageAllocator.hpp
new file mode 100644
index 00000000000..b907e50043d
--- /dev/null
+++ b/src/hotspot/share/gc/x/xPageAllocator.hpp
@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XPAGEALLOCATOR_HPP
+#define SHARE_GC_X_XPAGEALLOCATOR_HPP
+
+#include "gc/x/xAllocationFlags.hpp"
+#include "gc/x/xArray.hpp"
+#include "gc/x/xList.hpp"
+#include "gc/x/xLock.hpp"
+#include "gc/x/xPageCache.hpp"
+#include "gc/x/xPhysicalMemory.hpp"
+#include "gc/x/xSafeDelete.hpp"
+#include "gc/x/xVirtualMemory.hpp"
+
+class ThreadClosure;
+class VMStructs;
+class XPageAllocation;
+class XPageAllocatorStats;
+class XWorkers;
+class XUncommitter;
+class XUnmapper;
+
+class XPageAllocator {
+  friend class ::VMStructs;
+  friend class XUnmapper;
+  friend class XUncommitter;
+
+private:
+  mutable XLock              _lock;
+  XPageCache                 _cache;
+  XVirtualMemoryManager      _virtual;
+  XPhysicalMemoryManager     _physical;
+  const size_t               _min_capacity;
+  const size_t               _max_capacity;
+  volatile size_t            _current_max_capacity;
+  volatile size_t            _capacity;
+  volatile size_t            _claimed;
+  volatile size_t            _used;
+  size_t                     _used_high;
+  size_t                     _used_low;
+  ssize_t                    _reclaimed;
+  XList<XPageAllocation>     _stalled;
+  volatile uint64_t          _nstalled;
+  XList<XPageAllocation>     _satisfied;
+  XUnmapper*                 _unmapper;
+  XUncommitter*              _uncommitter;
+  mutable XSafeDelete<XPage> _safe_delete;
+  bool                       _initialized;
+
+  bool prime_cache(XWorkers* workers, size_t size);
+
+  size_t increase_capacity(size_t size);
+  void decrease_capacity(size_t size, bool set_max_capacity);
+
+  void increase_used(size_t size, bool relocation);
+  void decrease_used(size_t size, bool reclaimed);
+
+  bool commit_page(XPage* page);
+  void uncommit_page(XPage* page);
+
+  void map_page(const XPage* page) const;
+  void unmap_page(const XPage* page) const;
+
+  void destroy_page(XPage* page);
+
+  bool is_alloc_allowed(size_t size) const;
+
+  bool alloc_page_common_inner(uint8_t type, size_t size, XList<XPage>* pages);
+  bool alloc_page_common(XPageAllocation* allocation);
+  bool alloc_page_stall(XPageAllocation* allocation);
+  bool alloc_page_or_stall(XPageAllocation* allocation);
+  bool should_defragment(const XPage* page) const;
+  bool is_alloc_satisfied(XPageAllocation* allocation) const;
+  XPage* alloc_page_create(XPageAllocation* allocation);
+  XPage* alloc_page_finalize(XPageAllocation* allocation);
+  void alloc_page_failed(XPageAllocation* allocation);
+
+  void satisfy_stalled();
+
+  void free_page_inner(XPage* page, bool reclaimed);
+
+  size_t uncommit(uint64_t* timeout);
+
+public:
+  XPageAllocator(XWorkers* workers,
+                 size_t min_capacity,
+                 size_t initial_capacity,
+                 size_t max_capacity);
+
+  bool is_initialized() const;
+
+  size_t min_capacity() const;
+  size_t max_capacity() const;
+  size_t soft_max_capacity() const;
+  size_t capacity() const;
+  size_t used() const;
+  size_t unused() const;
+
+  XPageAllocatorStats stats() const;
+
+  void reset_statistics();
+
+  XPage* alloc_page(uint8_t type, size_t size, XAllocationFlags flags);
+  void free_page(XPage* page, bool reclaimed);
+  void free_pages(const XArray<XPage*>* pages, bool reclaimed);
+
+  void enable_deferred_delete() const;
+  void disable_deferred_delete() const;
+
+  void debug_map_page(const XPage* page) const;
+  void debug_unmap_page(const XPage* page) const;
+
+  bool has_alloc_stalled() const;
+  void check_out_of_memory();
+
+  void pages_do(XPageClosure* cl) const;
+
+  void threads_do(ThreadClosure* tc) const;
+};
+
+class XPageAllocatorStats {
+private:
+  size_t _min_capacity;
+  size_t _max_capacity;
+  size_t _soft_max_capacity;
+  size_t _current_max_capacity;
+  size_t _capacity;
+  size_t _used;
+  size_t _used_high;
+  size_t _used_low;
+  size_t _reclaimed;
+
+public:
+  XPageAllocatorStats(size_t min_capacity,
+                      size_t max_capacity,
+                      size_t soft_max_capacity,
+                      size_t capacity,
+                      size_t used,
+                      size_t used_high,
+                      size_t used_low,
+                      size_t reclaimed);
+
+  size_t min_capacity() const;
+  size_t max_capacity() const;
+  size_t soft_max_capacity() const;
+  size_t capacity() const;
+  size_t used() const;
+  size_t used_high() const;
+  size_t used_low() const;
+  size_t reclaimed() const;
+};
+
+#endif // SHARE_GC_X_XPAGEALLOCATOR_HPP
diff --git a/src/hotspot/share/gc/x/xPageAllocator.inline.hpp b/src/hotspot/share/gc/x/xPageAllocator.inline.hpp
new file mode 100644
index 00000000000..dbaf77f56a0
--- /dev/null
+++ b/src/hotspot/share/gc/x/xPageAllocator.inline.hpp
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XPAGEALLOCATOR_INLINE_HPP
+#define SHARE_GC_X_XPAGEALLOCATOR_INLINE_HPP
+
+#include "gc/x/xPageAllocator.hpp"
+
+inline XPageAllocatorStats::XPageAllocatorStats(size_t min_capacity,
+                                                size_t max_capacity,
+                                                size_t soft_max_capacity,
+                                                size_t capacity,
+                                                size_t used,
+                                                size_t used_high,
+                                                size_t used_low,
+                                                size_t reclaimed) :
+    _min_capacity(min_capacity),
+    _max_capacity(max_capacity),
+    _soft_max_capacity(soft_max_capacity),
+    _capacity(capacity),
+    _used(used),
+    _used_high(used_high),
+    _used_low(used_low),
+    _reclaimed(reclaimed) {}
+
+inline size_t XPageAllocatorStats::min_capacity() const {
+  return _min_capacity;
+}
+
+inline size_t XPageAllocatorStats::max_capacity() const {
+  return _max_capacity;
+}
+
+inline size_t XPageAllocatorStats::soft_max_capacity() const {
+  return _soft_max_capacity;
+}
+
+inline size_t XPageAllocatorStats::capacity() const {
+  return _capacity;
+}
+
+inline size_t XPageAllocatorStats::used() const {
+  return _used;
+}
+
+inline size_t XPageAllocatorStats::used_high() const {
+  return _used_high;
+}
+
+inline size_t XPageAllocatorStats::used_low() const {
+  return _used_low;
+}
+
+inline size_t XPageAllocatorStats::reclaimed() const {
+  return _reclaimed;
+}
+
+#endif // SHARE_GC_X_XPAGEALLOCATOR_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xPageCache.cpp b/src/hotspot/share/gc/x/xPageCache.cpp
new file mode 100644
index 00000000000..8f8a6636369
--- /dev/null
+++ b/src/hotspot/share/gc/x/xPageCache.cpp
@@ -0,0 +1,356 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xList.inline.hpp"
+#include "gc/x/xNUMA.hpp"
+#include "gc/x/xPage.inline.hpp"
+#include "gc/x/xPageCache.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xValue.inline.hpp"
+#include "memory/allocation.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/os.hpp"
+
+static const XStatCounter XCounterPageCacheHitL1("Memory", "Page Cache Hit L1", XStatUnitOpsPerSecond);
+static const XStatCounter XCounterPageCacheHitL2("Memory", "Page Cache Hit L2", XStatUnitOpsPerSecond);
+static const XStatCounter XCounterPageCacheHitL3("Memory", "Page Cache Hit L3", XStatUnitOpsPerSecond);
+static const XStatCounter XCounterPageCacheMiss("Memory", "Page Cache Miss", XStatUnitOpsPerSecond);
+
+class XPageCacheFlushClosure : public StackObj {
+  friend class XPageCache;
+
+protected:
+  const size_t _requested;
+  size_t       _flushed;
+
+public:
+  XPageCacheFlushClosure(size_t requested);
+  virtual bool do_page(const XPage* page) = 0;
+};
+
+XPageCacheFlushClosure::XPageCacheFlushClosure(size_t requested) :
+    _requested(requested),
+    _flushed(0) {}
+
+XPageCache::XPageCache() :
+    _small(),
+    _medium(),
+    _large(),
+    _last_commit(0) {}
+
+XPage* XPageCache::alloc_small_page() {
+  const uint32_t numa_id = XNUMA::id();
+  const uint32_t numa_count = XNUMA::count();
+
+  // Try NUMA local page cache
+  XPage* const l1_page = _small.get(numa_id).remove_first();
+  if (l1_page != NULL) {
+    XStatInc(XCounterPageCacheHitL1);
+    return l1_page;
+  }
+
+  // Try NUMA remote page cache(s)
+  uint32_t remote_numa_id = numa_id + 1;
+  const uint32_t remote_numa_count = numa_count - 1;
+  for (uint32_t i = 0; i < remote_numa_count; i++) {
+    if (remote_numa_id == numa_count) {
+      remote_numa_id = 0;
+    }
+
+    XPage* const l2_page = _small.get(remote_numa_id).remove_first();
+    if (l2_page != NULL) {
+      XStatInc(XCounterPageCacheHitL2);
+      return l2_page;
+    }
+
+    remote_numa_id++;
+  }
+
+  return NULL;
+}
+
+XPage* XPageCache::alloc_medium_page() {
+  XPage* const page = _medium.remove_first();
+  if (page != NULL) {
+    XStatInc(XCounterPageCacheHitL1);
+    return page;
+  }
+
+  return NULL;
+}
+
+XPage* XPageCache::alloc_large_page(size_t size) {
+  // Find a page with the right size
+  XListIterator<XPage> iter(&_large);
+  for (XPage* page; iter.next(&page);) {
+    if (size == page->size()) {
+      // Page found
+      _large.remove(page);
+      XStatInc(XCounterPageCacheHitL1);
+      return page;
+    }
+  }
+
+  return NULL;
+}
+
+XPage* XPageCache::alloc_oversized_medium_page(size_t size) {
+  if (size <= XPageSizeMedium) {
+    return _medium.remove_first();
+  }
+
+  return NULL;
+}
+
+XPage* XPageCache::alloc_oversized_large_page(size_t size) {
+  // Find a page that is large enough
+  XListIterator<XPage> iter(&_large);
+  for (XPage* page; iter.next(&page);) {
+    if (size <= page->size()) {
+      // Page found
+      _large.remove(page);
+      return page;
+    }
+  }
+
+  return NULL;
+}
+
+XPage* XPageCache::alloc_oversized_page(size_t size) {
+  XPage* page = alloc_oversized_large_page(size);
+  if (page == NULL) {
+    page = alloc_oversized_medium_page(size);
+  }
+
+  if (page != NULL) {
+    XStatInc(XCounterPageCacheHitL3);
+  }
+
+  return page;
+}
+
+XPage* XPageCache::alloc_page(uint8_t type, size_t size) {
+  XPage* page;
+
+  // Try allocate exact page
+  if (type == XPageTypeSmall) {
+    page = alloc_small_page();
+  } else if (type == XPageTypeMedium) {
+    page = alloc_medium_page();
+  } else {
+    page = alloc_large_page(size);
+  }
+
+  if (page == NULL) {
+    // Try allocate potentially oversized page
+    XPage* const oversized = alloc_oversized_page(size);
+    if (oversized != NULL) {
+      if (size < oversized->size()) {
+        // Split oversized page
+        page = oversized->split(type, size);
+
+        // Cache remainder
+        free_page(oversized);
+      } else {
+        // Re-type correctly sized page
+        page = oversized->retype(type);
+      }
+    }
+  }
+
+  if (page == NULL) {
+    XStatInc(XCounterPageCacheMiss);
+  }
+
+  return page;
+}
+
+void XPageCache::free_page(XPage* page) {
+  const uint8_t type = page->type();
+  if (type == XPageTypeSmall) {
+    _small.get(page->numa_id()).insert_first(page);
+  } else if (type == XPageTypeMedium) {
+    _medium.insert_first(page);
+  } else {
+    _large.insert_first(page);
+  }
+}
+
+bool XPageCache::flush_list_inner(XPageCacheFlushClosure* cl, XList<XPage>* from, XList<XPage>* to) {
+  XPage* const page = from->last();
+  if (page == NULL || !cl->do_page(page)) {
+    // Don't flush page
+    return false;
+  }
+
+  // Flush page
+  from->remove(page);
+  to->insert_last(page);
+  return true;
+}
+
+void XPageCache::flush_list(XPageCacheFlushClosure* cl, XList<XPage>* from, XList<XPage>* to) {
+  while (flush_list_inner(cl, from, to));
+}
+
+void XPageCache::flush_per_numa_lists(XPageCacheFlushClosure* cl, XPerNUMA<XList<XPage> >* from, XList<XPage>* to) {
+  const uint32_t numa_count = XNUMA::count();
+  uint32_t numa_done = 0;
+  uint32_t numa_next = 0;
+
+  // Flush lists round-robin
+  while (numa_done < numa_count) {
+    XList<XPage>* numa_list = from->addr(numa_next);
+    if (++numa_next == numa_count) {
+      numa_next = 0;
+    }
+
+    if (flush_list_inner(cl, numa_list, to)) {
+      // Not done
+      numa_done = 0;
+    } else {
+      // Done
+      numa_done++;
+    }
+  }
+}
+
+void XPageCache::flush(XPageCacheFlushClosure* cl, XList<XPage>* to) {
+  // Prefer flushing large, then medium and last small pages
+  flush_list(cl, &_large, to);
+  flush_list(cl, &_medium, to);
+  flush_per_numa_lists(cl, &_small, to);
+
+  if (cl->_flushed > cl->_requested) {
+    // Overflushed, re-insert part of last page into the cache
+    const size_t overflushed = cl->_flushed - cl->_requested;
+    XPage* const reinsert = to->last()->split(overflushed);
+    free_page(reinsert);
+    cl->_flushed -= overflushed;
+  }
+}
+
+class XPageCacheFlushForAllocationClosure : public XPageCacheFlushClosure {
+public:
+  XPageCacheFlushForAllocationClosure(size_t requested) :
+      XPageCacheFlushClosure(requested) {}
+
+  virtual bool do_page(const XPage* page) {
+    if (_flushed < _requested) {
+      // Flush page
+      _flushed += page->size();
+      return true;
+    }
+
+    // Don't flush page
+    return false;
+  }
+};
+
+void XPageCache::flush_for_allocation(size_t requested, XList<XPage>* to) {
+  XPageCacheFlushForAllocationClosure cl(requested);
+  flush(&cl, to);
+}
+
+class XPageCacheFlushForUncommitClosure : public XPageCacheFlushClosure {
+private:
+  const uint64_t _now;
+  uint64_t*      _timeout;
+
+public:
+  XPageCacheFlushForUncommitClosure(size_t requested, uint64_t now, uint64_t* timeout) :
+      XPageCacheFlushClosure(requested),
+      _now(now),
+      _timeout(timeout) {
+    // Set initial timeout
+    *_timeout = ZUncommitDelay;
+  }
+
+  virtual bool do_page(const XPage* page) {
+    const uint64_t expires = page->last_used() + ZUncommitDelay;
+    if (expires > _now) {
+      // Don't flush page, record shortest non-expired timeout
+      *_timeout = MIN2(*_timeout, expires - _now);
+      return false;
+    }
+
+    if (_flushed >= _requested) {
+      // Don't flush page, requested amount flushed
+      return false;
+    }
+
+    // Flush page
+    _flushed += page->size();
+    return true;
+  }
+};
+
+size_t XPageCache::flush_for_uncommit(size_t requested, XList<XPage>* to, uint64_t* timeout) {
+  const uint64_t now = os::elapsedTime();
+  const uint64_t expires = _last_commit + ZUncommitDelay;
+  if (expires > now) {
+    // Delay uncommit, set next timeout
+    *timeout = expires - now;
+    return 0;
+  }
+
+  if (requested == 0) {
+    // Nothing to flush, set next timeout
+    *timeout = ZUncommitDelay;
+    return 0;
+  }
+
+  XPageCacheFlushForUncommitClosure cl(requested, now, timeout);
+  flush(&cl, to);
+
+  return cl._flushed;
+}
+
+void XPageCache::set_last_commit() {
+  _last_commit = ceil(os::elapsedTime());
+}
+
+void XPageCache::pages_do(XPageClosure* cl) const {
+  // Small
+  XPerNUMAConstIterator<XList<XPage> > iter_numa(&_small);
+  for (const XList<XPage>* list; iter_numa.next(&list);) {
+    XListIterator<XPage> iter_small(list);
+    for (XPage* page; iter_small.next(&page);) {
+      cl->do_page(page);
+    }
+  }
+
+  // Medium
+  XListIterator<XPage> iter_medium(&_medium);
+  for (XPage* page; iter_medium.next(&page);) {
+    cl->do_page(page);
+  }
+
+  // Large
+  XListIterator<XPage> iter_large(&_large);
+  for (XPage* page; iter_large.next(&page);) {
+    cl->do_page(page);
+  }
+}
diff --git a/src/hotspot/share/gc/x/xPageCache.hpp b/src/hotspot/share/gc/x/xPageCache.hpp
new file mode 100644
index 00000000000..9ed80a933f4
--- /dev/null
+++ b/src/hotspot/share/gc/x/xPageCache.hpp
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XPAGECACHE_HPP
+#define SHARE_GC_X_XPAGECACHE_HPP
+
+#include "gc/x/xList.hpp"
+#include "gc/x/xPage.hpp"
+#include "gc/x/xValue.hpp"
+
+class XPageCacheFlushClosure;
+
+class XPageCache {
+private:
+  XPerNUMA<XList<XPage> > _small;
+  XList<XPage>            _medium;
+  XList<XPage>            _large;
+  uint64_t                _last_commit;
+
+  XPage* alloc_small_page();
+  XPage* alloc_medium_page();
+  XPage* alloc_large_page(size_t size);
+
+  XPage* alloc_oversized_medium_page(size_t size);
+  XPage* alloc_oversized_large_page(size_t size);
+  XPage* alloc_oversized_page(size_t size);
+
+  bool flush_list_inner(XPageCacheFlushClosure* cl, XList<XPage>* from, XList<XPage>* to);
+  void flush_list(XPageCacheFlushClosure* cl, XList<XPage>* from, XList<XPage>* to);
+  void flush_per_numa_lists(XPageCacheFlushClosure* cl, XPerNUMA<XList<XPage> >* from, XList<XPage>* to);
+  void flush(XPageCacheFlushClosure* cl, XList<XPage>* to);
+
+public:
+  XPageCache();
+
+  XPage* alloc_page(uint8_t type, size_t size);
+  void free_page(XPage* page);
+
+  void flush_for_allocation(size_t requested, XList<XPage>* to);
+  size_t flush_for_uncommit(size_t requested, XList<XPage>* to, uint64_t* timeout);
+
+  void set_last_commit();
+
+  void pages_do(XPageClosure* cl) const;
+};
+
+#endif // SHARE_GC_X_XPAGECACHE_HPP
diff --git a/src/hotspot/share/gc/x/xPageTable.cpp b/src/hotspot/share/gc/x/xPageTable.cpp
new file mode 100644
index 00000000000..6cdb7c929e1
--- /dev/null
+++ b/src/hotspot/share/gc/x/xPageTable.cpp
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xGranuleMap.inline.hpp"
+#include "gc/x/xPage.inline.hpp"
+#include "gc/x/xPageTable.inline.hpp"
+#include "runtime/orderAccess.hpp"
+#include "utilities/debug.hpp"
+
+XPageTable::XPageTable() :
+    _map(XAddressOffsetMax) {}
+
+void XPageTable::insert(XPage* page) {
+  const uintptr_t offset = page->start();
+  const size_t size = page->size();
+
+  // Make sure a newly created page is
+  // visible before updating the page table.
+  OrderAccess::storestore();
+
+  assert(_map.get(offset) == NULL, "Invalid entry");
+  _map.put(offset, size, page);
+}
+
+void XPageTable::remove(XPage* page) {
+  const uintptr_t offset = page->start();
+  const size_t size = page->size();
+
+  assert(_map.get(offset) == page, "Invalid entry");
+  _map.put(offset, size, NULL);
+}
diff --git a/src/hotspot/share/gc/x/xPageTable.hpp b/src/hotspot/share/gc/x/xPageTable.hpp
new file mode 100644
index 00000000000..958dd735557
--- /dev/null
+++ b/src/hotspot/share/gc/x/xPageTable.hpp
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XPAGETABLE_HPP
+#define SHARE_GC_X_XPAGETABLE_HPP
+
+#include "gc/x/xGranuleMap.hpp"
+#include "memory/allocation.hpp"
+
+class VMStructs;
+class XPage;
+class XPageTableIterator;
+
+class XPageTable {
+  friend class ::VMStructs;
+  friend class XPageTableIterator;
+
+private:
+  XGranuleMap<XPage*> _map;
+
+public:
+  XPageTable();
+
+  XPage* get(uintptr_t addr) const;
+
+  void insert(XPage* page);
+  void remove(XPage* page);
+};
+
+class XPageTableIterator : public StackObj {
+private:
+  XGranuleMapIterator<XPage*> _iter;
+  XPage*                      _prev;
+
+public:
+  XPageTableIterator(const XPageTable* page_table);
+
+  bool next(XPage** page);
+};
+
+#endif // SHARE_GC_X_XPAGETABLE_HPP
diff --git a/src/hotspot/share/gc/x/xPageTable.inline.hpp b/src/hotspot/share/gc/x/xPageTable.inline.hpp
new file mode 100644
index 00000000000..c4f30d3e9c3
--- /dev/null
+++ b/src/hotspot/share/gc/x/xPageTable.inline.hpp
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XPAGETABLE_INLINE_HPP
+#define SHARE_GC_X_XPAGETABLE_INLINE_HPP
+
+#include "gc/x/xPageTable.hpp"
+
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xGranuleMap.inline.hpp"
+
+inline XPage* XPageTable::get(uintptr_t addr) const {
+  assert(!XAddress::is_null(addr), "Invalid address");
+  return _map.get(XAddress::offset(addr));
+}
+
+inline XPageTableIterator::XPageTableIterator(const XPageTable* page_table) :
+    _iter(&page_table->_map),
+    _prev(NULL) {}
+
+inline bool XPageTableIterator::next(XPage** page) {
+  for (XPage* entry; _iter.next(&entry);) {
+    if (entry != NULL && entry != _prev) {
+      // Next page found
+      *page = _prev = entry;
+      return true;
+    }
+  }
+
+  // No more pages
+  return false;
+}
+
+#endif // SHARE_GC_X_XPAGETABLE_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xPhysicalMemory.cpp b/src/hotspot/share/gc/x/xPhysicalMemory.cpp
new file mode 100644
index 00000000000..20902cc05bc
--- /dev/null
+++ b/src/hotspot/share/gc/x/xPhysicalMemory.cpp
@@ -0,0 +1,435 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xArray.inline.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xLargePages.inline.hpp"
+#include "gc/x/xNUMA.inline.hpp"
+#include "gc/x/xPhysicalMemory.inline.hpp"
+#include "logging/log.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/globals_extension.hpp"
+#include "runtime/init.hpp"
+#include "runtime/os.hpp"
+#include "services/memTracker.hpp"
+#include "utilities/align.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+XPhysicalMemory::XPhysicalMemory() :
+    _segments() {}
+
+XPhysicalMemory::XPhysicalMemory(const XPhysicalMemorySegment& segment) :
+    _segments() {
+  add_segment(segment);
+}
+
+XPhysicalMemory::XPhysicalMemory(const XPhysicalMemory& pmem) :
+    _segments() {
+  add_segments(pmem);
+}
+
+const XPhysicalMemory& XPhysicalMemory::operator=(const XPhysicalMemory& pmem) {
+  // Free segments
+  _segments.clear_and_deallocate();
+
+  // Copy segments
+  add_segments(pmem);
+
+  return *this;
+}
+
+size_t XPhysicalMemory::size() const {
+  size_t size = 0;
+
+  for (int i = 0; i < _segments.length(); i++) {
+    size += _segments.at(i).size();
+  }
+
+  return size;
+}
+
+void XPhysicalMemory::insert_segment(int index, uintptr_t start, size_t size, bool committed) {
+  _segments.insert_before(index, XPhysicalMemorySegment(start, size, committed));
+}
+
+void XPhysicalMemory::replace_segment(int index, uintptr_t start, size_t size, bool committed) {
+  _segments.at_put(index, XPhysicalMemorySegment(start, size, committed));
+}
+
+void XPhysicalMemory::remove_segment(int index) {
+  _segments.remove_at(index);
+}
+
+void XPhysicalMemory::add_segments(const XPhysicalMemory& pmem) {
+  for (int i = 0; i < pmem.nsegments(); i++) {
+    add_segment(pmem.segment(i));
+  }
+}
+
+void XPhysicalMemory::remove_segments() {
+  _segments.clear_and_deallocate();
+}
+
+static bool is_mergable(const XPhysicalMemorySegment& before, const XPhysicalMemorySegment& after) {
+  return before.end() == after.start() && before.is_committed() == after.is_committed();
+}
+
+void XPhysicalMemory::add_segment(const XPhysicalMemorySegment& segment) {
+  // Insert segments in address order, merge segments when possible
+  for (int i = _segments.length(); i > 0; i--) {
+    const int current = i - 1;
+
+    if (_segments.at(current).end() <= segment.start()) {
+      if (is_mergable(_segments.at(current), segment)) {
+        if (current + 1 < _segments.length() && is_mergable(segment, _segments.at(current + 1))) {
+          // Merge with end of current segment and start of next segment
+          const size_t start = _segments.at(current).start();
+          const size_t size = _segments.at(current).size() + segment.size() + _segments.at(current + 1).size();
+          replace_segment(current, start, size, segment.is_committed());
+          remove_segment(current + 1);
+          return;
+        }
+
+        // Merge with end of current segment
+        const size_t start = _segments.at(current).start();
+        const size_t size = _segments.at(current).size() + segment.size();
+        replace_segment(current, start, size, segment.is_committed());
+        return;
+      } else if (current + 1 < _segments.length() && is_mergable(segment, _segments.at(current + 1))) {
+        // Merge with start of next segment
+        const size_t start = segment.start();
+        const size_t size = segment.size() + _segments.at(current + 1).size();
+        replace_segment(current + 1, start, size, segment.is_committed());
+        return;
+      }
+
+      // Insert after current segment
+      insert_segment(current + 1, segment.start(), segment.size(), segment.is_committed());
+      return;
+    }
+  }
+
+  if (_segments.length() > 0 && is_mergable(segment, _segments.at(0))) {
+    // Merge with start of first segment
+    const size_t start = segment.start();
+    const size_t size = segment.size() + _segments.at(0).size();
+    replace_segment(0, start, size, segment.is_committed());
+    return;
+  }
+
+  // Insert before first segment
+  insert_segment(0, segment.start(), segment.size(), segment.is_committed());
+}
+
+bool XPhysicalMemory::commit_segment(int index, size_t size) {
+  assert(size <= _segments.at(index).size(), "Invalid size");
+  assert(!_segments.at(index).is_committed(), "Invalid state");
+
+  if (size == _segments.at(index).size()) {
+    // Completely committed
+    _segments.at(index).set_committed(true);
+    return true;
+  }
+
+  if (size > 0) {
+    // Partially committed, split segment
+    insert_segment(index + 1, _segments.at(index).start() + size, _segments.at(index).size() - size, false /* committed */);
+    replace_segment(index, _segments.at(index).start(), size, true /* committed */);
+  }
+
+  return false;
+}
+
+bool XPhysicalMemory::uncommit_segment(int index, size_t size) {
+  assert(size <= _segments.at(index).size(), "Invalid size");
+  assert(_segments.at(index).is_committed(), "Invalid state");
+
+  if (size == _segments.at(index).size()) {
+    // Completely uncommitted
+    _segments.at(index).set_committed(false);
+    return true;
+  }
+
+  if (size > 0) {
+    // Partially uncommitted, split segment
+    insert_segment(index + 1, _segments.at(index).start() + size, _segments.at(index).size() - size, true /* committed */);
+    replace_segment(index, _segments.at(index).start(), size, false /* committed */);
+  }
+
+  return false;
+}
+
+XPhysicalMemory XPhysicalMemory::split(size_t size) {
+  XPhysicalMemory pmem;
+  int nsegments = 0;
+
+  for (int i = 0; i < _segments.length(); i++) {
+    const XPhysicalMemorySegment& segment = _segments.at(i);
+    if (pmem.size() < size) {
+      if (pmem.size() + segment.size() <= size) {
+        // Transfer segment
+        pmem.add_segment(segment);
+      } else {
+        // Split segment
+        const size_t split_size = size - pmem.size();
+        pmem.add_segment(XPhysicalMemorySegment(segment.start(), split_size, segment.is_committed()));
+        _segments.at_put(nsegments++, XPhysicalMemorySegment(segment.start() + split_size, segment.size() - split_size, segment.is_committed()));
+      }
+    } else {
+      // Keep segment
+      _segments.at_put(nsegments++, segment);
+    }
+  }
+
+  _segments.trunc_to(nsegments);
+
+  return pmem;
+}
+
+XPhysicalMemory XPhysicalMemory::split_committed() {
+  XPhysicalMemory pmem;
+  int nsegments = 0;
+
+  for (int i = 0; i < _segments.length(); i++) {
+    const XPhysicalMemorySegment& segment = _segments.at(i);
+    if (segment.is_committed()) {
+      // Transfer segment
+      pmem.add_segment(segment);
+    } else {
+      // Keep segment
+      _segments.at_put(nsegments++, segment);
+    }
+  }
+
+  _segments.trunc_to(nsegments);
+
+  return pmem;
+}
+
+XPhysicalMemoryManager::XPhysicalMemoryManager(size_t max_capacity) :
+    _backing(max_capacity) {
+  // Make the whole range free
+  _manager.free(0, max_capacity);
+}
+
+bool XPhysicalMemoryManager::is_initialized() const {
+  return _backing.is_initialized();
+}
+
+void XPhysicalMemoryManager::warn_commit_limits(size_t max_capacity) const {
+  _backing.warn_commit_limits(max_capacity);
+}
+
+void XPhysicalMemoryManager::try_enable_uncommit(size_t min_capacity, size_t max_capacity) {
+  assert(!is_init_completed(), "Invalid state");
+
+  // If uncommit is not explicitly disabled, max capacity is greater than
+  // min capacity, and uncommit is supported by the platform, then uncommit
+  // will be enabled.
+  if (!ZUncommit) {
+    log_info_p(gc, init)("Uncommit: Disabled");
+    return;
+  }
+
+  if (max_capacity == min_capacity) {
+    log_info_p(gc, init)("Uncommit: Implicitly Disabled (-Xms equals -Xmx)");
+    FLAG_SET_ERGO(ZUncommit, false);
+    return;
+  }
+
+  // Test if uncommit is supported by the operating system by committing
+  // and then uncommitting a granule.
+  XPhysicalMemory pmem(XPhysicalMemorySegment(0, XGranuleSize, false /* committed */));
+  if (!commit(pmem) || !uncommit(pmem)) {
+    log_info_p(gc, init)("Uncommit: Implicitly Disabled (Not supported by operating system)");
+    FLAG_SET_ERGO(ZUncommit, false);
+    return;
+  }
+
+  log_info_p(gc, init)("Uncommit: Enabled");
+  log_info_p(gc, init)("Uncommit Delay: " UINTX_FORMAT "s", ZUncommitDelay);
+}
+
+void XPhysicalMemoryManager::nmt_commit(uintptr_t offset, size_t size) const {
+  // From an NMT point of view we treat the first heap view (marked0) as committed
+  const uintptr_t addr = XAddress::marked0(offset);
+  MemTracker::record_virtual_memory_commit((void*)addr, size, CALLER_PC);
+}
+
+void XPhysicalMemoryManager::nmt_uncommit(uintptr_t offset, size_t size) const {
+  if (MemTracker::enabled()) {
+    const uintptr_t addr = XAddress::marked0(offset);
+    Tracker tracker(Tracker::uncommit);
+    tracker.record((address)addr, size);
+  }
+}
+
+void XPhysicalMemoryManager::alloc(XPhysicalMemory& pmem, size_t size) {
+  assert(is_aligned(size, XGranuleSize), "Invalid size");
+
+  // Allocate segments
+  while (size > 0) {
+    size_t allocated = 0;
+    const uintptr_t start = _manager.alloc_low_address_at_most(size, &allocated);
+    assert(start != UINTPTR_MAX, "Allocation should never fail");
+    pmem.add_segment(XPhysicalMemorySegment(start, allocated, false /* committed */));
+    size -= allocated;
+  }
+}
+
+void XPhysicalMemoryManager::free(const XPhysicalMemory& pmem) {
+  // Free segments
+  for (int i = 0; i < pmem.nsegments(); i++) {
+    const XPhysicalMemorySegment& segment = pmem.segment(i);
+    _manager.free(segment.start(), segment.size());
+  }
+}
+
+bool XPhysicalMemoryManager::commit(XPhysicalMemory& pmem) {
+  // Commit segments
+  for (int i = 0; i < pmem.nsegments(); i++) {
+    const XPhysicalMemorySegment& segment = pmem.segment(i);
+    if (segment.is_committed()) {
+      // Segment already committed
+      continue;
+    }
+
+    // Commit segment
+    const size_t committed = _backing.commit(segment.start(), segment.size());
+    if (!pmem.commit_segment(i, committed)) {
+      // Failed or partially failed
+      return false;
+    }
+  }
+
+  // Success
+  return true;
+}
+
+bool XPhysicalMemoryManager::uncommit(XPhysicalMemory& pmem) {
+  // Commit segments
+  for (int i = 0; i < pmem.nsegments(); i++) {
+    const XPhysicalMemorySegment& segment = pmem.segment(i);
+    if (!segment.is_committed()) {
+      // Segment already uncommitted
+      continue;
+    }
+
+    // Uncommit segment
+    const size_t uncommitted = _backing.uncommit(segment.start(), segment.size());
+    if (!pmem.uncommit_segment(i, uncommitted)) {
+      // Failed or partially failed
+      return false;
+    }
+  }
+
+  // Success
+  return true;
+}
+
+void XPhysicalMemoryManager::pretouch_view(uintptr_t addr, size_t size) const {
+  const size_t page_size = XLargePages::is_explicit() ? XGranuleSize : os::vm_page_size();
+  os::pretouch_memory((void*)addr, (void*)(addr + size), page_size);
+}
+
+void XPhysicalMemoryManager::map_view(uintptr_t addr, const XPhysicalMemory& pmem) const {
+  size_t size = 0;
+
+  // Map segments
+  for (int i = 0; i < pmem.nsegments(); i++) {
+    const XPhysicalMemorySegment& segment = pmem.segment(i);
+    _backing.map(addr + size, segment.size(), segment.start());
+    size += segment.size();
+  }
+
+  // Setup NUMA interleaving for large pages
+  if (XNUMA::is_enabled() && XLargePages::is_explicit()) {
+    // To get granule-level NUMA interleaving when using large pages,
+    // we simply let the kernel interleave the memory for us at page
+    // fault time.
+    os::numa_make_global((char*)addr, size);
+  }
+}
+
+void XPhysicalMemoryManager::unmap_view(uintptr_t addr, size_t size) const {
+  _backing.unmap(addr, size);
+}
+
+void XPhysicalMemoryManager::pretouch(uintptr_t offset, size_t size) const {
+  if (ZVerifyViews) {
+    // Pre-touch good view
+    pretouch_view(XAddress::good(offset), size);
+  } else {
+    // Pre-touch all views
+    pretouch_view(XAddress::marked0(offset), size);
+    pretouch_view(XAddress::marked1(offset), size);
+    pretouch_view(XAddress::remapped(offset), size);
+  }
+}
+
+void XPhysicalMemoryManager::map(uintptr_t offset, const XPhysicalMemory& pmem) const {
+  const size_t size = pmem.size();
+
+  if (ZVerifyViews) {
+    // Map good view
+    map_view(XAddress::good(offset), pmem);
+  } else {
+    // Map all views
+    map_view(XAddress::marked0(offset), pmem);
+    map_view(XAddress::marked1(offset), pmem);
+    map_view(XAddress::remapped(offset), pmem);
+  }
+
+  nmt_commit(offset, size);
+}
+
+void XPhysicalMemoryManager::unmap(uintptr_t offset, size_t size) const {
+  nmt_uncommit(offset, size);
+
+  if (ZVerifyViews) {
+    // Unmap good view
+    unmap_view(XAddress::good(offset), size);
+  } else {
+    // Unmap all views
+    unmap_view(XAddress::marked0(offset), size);
+    unmap_view(XAddress::marked1(offset), size);
+    unmap_view(XAddress::remapped(offset), size);
+  }
+}
+
+void XPhysicalMemoryManager::debug_map(uintptr_t offset, const XPhysicalMemory& pmem) const {
+  // Map good view
+  assert(ZVerifyViews, "Should be enabled");
+  map_view(XAddress::good(offset), pmem);
+}
+
+void XPhysicalMemoryManager::debug_unmap(uintptr_t offset, size_t size) const {
+  // Unmap good view
+  assert(ZVerifyViews, "Should be enabled");
+  unmap_view(XAddress::good(offset), size);
+}
diff --git a/src/hotspot/share/gc/x/xPhysicalMemory.hpp b/src/hotspot/share/gc/x/xPhysicalMemory.hpp
new file mode 100644
index 00000000000..26d8ed9bb96
--- /dev/null
+++ b/src/hotspot/share/gc/x/xPhysicalMemory.hpp
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XPHYSICALMEMORY_HPP
+#define SHARE_GC_X_XPHYSICALMEMORY_HPP
+
+#include "gc/x/xArray.hpp"
+#include "gc/x/xMemory.hpp"
+#include "memory/allocation.hpp"
+#include OS_HEADER(gc/x/xPhysicalMemoryBacking)
+
+class XPhysicalMemorySegment : public CHeapObj<mtGC> {
+private:
+  uintptr_t _start;
+  uintptr_t _end;
+  bool      _committed;
+
+public:
+  XPhysicalMemorySegment();
+  XPhysicalMemorySegment(uintptr_t start, size_t size, bool committed);
+
+  uintptr_t start() const;
+  uintptr_t end() const;
+  size_t size() const;
+
+  bool is_committed() const;
+  void set_committed(bool committed);
+};
+
+class XPhysicalMemory {
+private:
+  XArray<XPhysicalMemorySegment> _segments;
+
+  void insert_segment(int index, uintptr_t start, size_t size, bool committed);
+  void replace_segment(int index, uintptr_t start, size_t size, bool committed);
+  void remove_segment(int index);
+
+public:
+  XPhysicalMemory();
+  XPhysicalMemory(const XPhysicalMemorySegment& segment);
+  XPhysicalMemory(const XPhysicalMemory& pmem);
+  const XPhysicalMemory& operator=(const XPhysicalMemory& pmem);
+
+  bool is_null() const;
+  size_t size() const;
+
+  int nsegments() const;
+  const XPhysicalMemorySegment& segment(int index) const;
+
+  void add_segments(const XPhysicalMemory& pmem);
+  void remove_segments();
+
+  void add_segment(const XPhysicalMemorySegment& segment);
+  bool commit_segment(int index, size_t size);
+  bool uncommit_segment(int index, size_t size);
+
+  XPhysicalMemory split(size_t size);
+  XPhysicalMemory split_committed();
+};
+
+class XPhysicalMemoryManager {
+private:
+  XPhysicalMemoryBacking _backing;
+  XMemoryManager         _manager;
+
+  void nmt_commit(uintptr_t offset, size_t size) const;
+  void nmt_uncommit(uintptr_t offset, size_t size) const;
+
+  void pretouch_view(uintptr_t addr, size_t size) const;
+  void map_view(uintptr_t addr, const XPhysicalMemory& pmem) const;
+  void unmap_view(uintptr_t addr, size_t size) const;
+
+public:
+  XPhysicalMemoryManager(size_t max_capacity);
+
+  bool is_initialized() const;
+
+  void warn_commit_limits(size_t max_capacity) const;
+  void try_enable_uncommit(size_t min_capacity, size_t max_capacity);
+
+  void alloc(XPhysicalMemory& pmem, size_t size);
+  void free(const XPhysicalMemory& pmem);
+
+  bool commit(XPhysicalMemory& pmem);
+  bool uncommit(XPhysicalMemory& pmem);
+
+  void pretouch(uintptr_t offset, size_t size) const;
+
+  void map(uintptr_t offset, const XPhysicalMemory& pmem) const;
+  void unmap(uintptr_t offset, size_t size) const;
+
+  void debug_map(uintptr_t offset, const XPhysicalMemory& pmem) const;
+  void debug_unmap(uintptr_t offset, size_t size) const;
+};
+
+#endif // SHARE_GC_X_XPHYSICALMEMORY_HPP
diff --git a/src/hotspot/share/gc/x/xPhysicalMemory.inline.hpp b/src/hotspot/share/gc/x/xPhysicalMemory.inline.hpp
new file mode 100644
index 00000000000..70f38e2abdb
--- /dev/null
+++ b/src/hotspot/share/gc/x/xPhysicalMemory.inline.hpp
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XPHYSICALMEMORY_INLINE_HPP
+#define SHARE_GC_X_XPHYSICALMEMORY_INLINE_HPP
+
+#include "gc/x/xPhysicalMemory.hpp"
+
+#include "gc/x/xAddress.inline.hpp"
+#include "utilities/debug.hpp"
+
+inline XPhysicalMemorySegment::XPhysicalMemorySegment() :
+    _start(UINTPTR_MAX),
+    _end(UINTPTR_MAX),
+    _committed(false) {}
+
+inline XPhysicalMemorySegment::XPhysicalMemorySegment(uintptr_t start, size_t size, bool committed) :
+    _start(start),
+    _end(start + size),
+    _committed(committed) {}
+
+inline uintptr_t XPhysicalMemorySegment::start() const {
+  return _start;
+}
+
+inline uintptr_t XPhysicalMemorySegment::end() const {
+  return _end;
+}
+
+inline size_t XPhysicalMemorySegment::size() const {
+  return _end - _start;
+}
+
+inline bool XPhysicalMemorySegment::is_committed() const {
+  return _committed;
+}
+
+inline void XPhysicalMemorySegment::set_committed(bool committed) {
+  _committed = committed;
+}
+
+inline bool XPhysicalMemory::is_null() const {
+  return _segments.length() == 0;
+}
+
+inline int XPhysicalMemory::nsegments() const {
+  return _segments.length();
+}
+
+inline const XPhysicalMemorySegment& XPhysicalMemory::segment(int index) const {
+  return _segments.at(index);
+}
+
+#endif // SHARE_GC_X_XPHYSICALMEMORY_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xReferenceProcessor.cpp b/src/hotspot/share/gc/x/xReferenceProcessor.cpp
new file mode 100644
index 00000000000..4d6f05e5922
--- /dev/null
+++ b/src/hotspot/share/gc/x/xReferenceProcessor.cpp
@@ -0,0 +1,459 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "classfile/javaClasses.inline.hpp"
+#include "gc/shared/referencePolicy.hpp"
+#include "gc/shared/referenceProcessorStats.hpp"
+#include "gc/x/xHeap.inline.hpp"
+#include "gc/x/xReferenceProcessor.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xTask.hpp"
+#include "gc/x/xTracer.inline.hpp"
+#include "gc/x/xValue.inline.hpp"
+#include "memory/universe.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/mutexLocker.hpp"
+#include "runtime/os.hpp"
+
+static const XStatSubPhase XSubPhaseConcurrentReferencesProcess("Concurrent References Process");
+static const XStatSubPhase XSubPhaseConcurrentReferencesEnqueue("Concurrent References Enqueue");
+
+static ReferenceType reference_type(oop reference) {
+  return InstanceKlass::cast(reference->klass())->reference_type();
+}
+
+static const char* reference_type_name(ReferenceType type) {
+  switch (type) {
+  case REF_SOFT:
+    return "Soft";
+
+  case REF_WEAK:
+    return "Weak";
+
+  case REF_FINAL:
+    return "Final";
+
+  case REF_PHANTOM:
+    return "Phantom";
+
+  default:
+    ShouldNotReachHere();
+    return "Unknown";
+  }
+}
+
+static volatile oop* reference_referent_addr(oop reference) {
+  return (volatile oop*)java_lang_ref_Reference::referent_addr_raw(reference);
+}
+
+static oop reference_referent(oop reference) {
+  return Atomic::load(reference_referent_addr(reference));
+}
+
+static void reference_clear_referent(oop reference) {
+  java_lang_ref_Reference::clear_referent_raw(reference);
+}
+
+static oop* reference_discovered_addr(oop reference) {
+  return (oop*)java_lang_ref_Reference::discovered_addr_raw(reference);
+}
+
+static oop reference_discovered(oop reference) {
+  return *reference_discovered_addr(reference);
+}
+
+static void reference_set_discovered(oop reference, oop discovered) {
+  java_lang_ref_Reference::set_discovered_raw(reference, discovered);
+}
+
+static oop* reference_next_addr(oop reference) {
+  return (oop*)java_lang_ref_Reference::next_addr_raw(reference);
+}
+
+static oop reference_next(oop reference) {
+  return *reference_next_addr(reference);
+}
+
+static void reference_set_next(oop reference, oop next) {
+  java_lang_ref_Reference::set_next_raw(reference, next);
+}
+
+static void soft_reference_update_clock() {
+  const jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC;
+  java_lang_ref_SoftReference::set_clock(now);
+}
+
+XReferenceProcessor::XReferenceProcessor(XWorkers* workers) :
+    _workers(workers),
+    _soft_reference_policy(NULL),
+    _encountered_count(),
+    _discovered_count(),
+    _enqueued_count(),
+    _discovered_list(NULL),
+    _pending_list(NULL),
+    _pending_list_tail(_pending_list.addr()) {}
+
+void XReferenceProcessor::set_soft_reference_policy(bool clear) {
+  static AlwaysClearPolicy always_clear_policy;
+  static LRUMaxHeapPolicy lru_max_heap_policy;
+
+  if (clear) {
+    log_info(gc, ref)("Clearing All SoftReferences");
+    _soft_reference_policy = &always_clear_policy;
+  } else {
+    _soft_reference_policy = &lru_max_heap_policy;
+  }
+
+  _soft_reference_policy->setup();
+}
+
+bool XReferenceProcessor::is_inactive(oop reference, oop referent, ReferenceType type) const {
+  if (type == REF_FINAL) {
+    // A FinalReference is inactive if its next field is non-null. An application can't
+    // call enqueue() or clear() on a FinalReference.
+    return reference_next(reference) != NULL;
+  } else {
+    // A non-FinalReference is inactive if the referent is null. The referent can only
+    // be null if the application called Reference.enqueue() or Reference.clear().
+    return referent == NULL;
+  }
+}
+
+bool XReferenceProcessor::is_strongly_live(oop referent) const {
+  return XHeap::heap()->is_object_strongly_live(XOop::to_address(referent));
+}
+
+bool XReferenceProcessor::is_softly_live(oop reference, ReferenceType type) const {
+  if (type != REF_SOFT) {
+    // Not a SoftReference
+    return false;
+  }
+
+  // Ask SoftReference policy
+  const jlong clock = java_lang_ref_SoftReference::clock();
+  assert(clock != 0, "Clock not initialized");
+  assert(_soft_reference_policy != NULL, "Policy not initialized");
+  return !_soft_reference_policy->should_clear_reference(reference, clock);
+}
+
+bool XReferenceProcessor::should_discover(oop reference, ReferenceType type) const {
+  volatile oop* const referent_addr = reference_referent_addr(reference);
+  const oop referent = XBarrier::weak_load_barrier_on_oop_field(referent_addr);
+
+  if (is_inactive(reference, referent, type)) {
+    return false;
+  }
+
+  if (is_strongly_live(referent)) {
+    return false;
+  }
+
+  if (is_softly_live(reference, type)) {
+    return false;
+  }
+
+  // PhantomReferences with finalizable marked referents should technically not have
+  // to be discovered. However, InstanceRefKlass::oop_oop_iterate_ref_processing()
+  // does not know about the finalizable mark concept, and will therefore mark
+  // referents in non-discovered PhantomReferences as strongly live. To prevent
+  // this, we always discover PhantomReferences with finalizable marked referents.
+  // They will automatically be dropped during the reference processing phase.
+  return true;
+}
+
+bool XReferenceProcessor::should_drop(oop reference, ReferenceType type) const {
+  const oop referent = reference_referent(reference);
+  if (referent == NULL) {
+    // Reference has been cleared, by a call to Reference.enqueue()
+    // or Reference.clear() from the application, which means we
+    // should drop the reference.
+    return true;
+  }
+
+  // Check if the referent is still alive, in which case we should
+  // drop the reference.
+  if (type == REF_PHANTOM) {
+    return XBarrier::is_alive_barrier_on_phantom_oop(referent);
+  } else {
+    return XBarrier::is_alive_barrier_on_weak_oop(referent);
+  }
+}
+
+void XReferenceProcessor::keep_alive(oop reference, ReferenceType type) const {
+  volatile oop* const p = reference_referent_addr(reference);
+  if (type == REF_PHANTOM) {
+    XBarrier::keep_alive_barrier_on_phantom_oop_field(p);
+  } else {
+    XBarrier::keep_alive_barrier_on_weak_oop_field(p);
+  }
+}
+
+void XReferenceProcessor::make_inactive(oop reference, ReferenceType type) const {
+  if (type == REF_FINAL) {
+    // Don't clear referent. It is needed by the Finalizer thread to make the call
+    // to finalize(). A FinalReference is instead made inactive by self-looping the
+    // next field. An application can't call FinalReference.enqueue(), so there is
+    // no race to worry about when setting the next field.
+    assert(reference_next(reference) == NULL, "Already inactive");
+    reference_set_next(reference, reference);
+  } else {
+    // Clear referent
+    reference_clear_referent(reference);
+  }
+}
+
+void XReferenceProcessor::discover(oop reference, ReferenceType type) {
+  log_trace(gc, ref)("Discovered Reference: " PTR_FORMAT " (%s)", p2i(reference), reference_type_name(type));
+
+  // Update statistics
+  _discovered_count.get()[type]++;
+
+  if (type == REF_FINAL) {
+    // Mark referent (and its reachable subgraph) finalizable. This avoids
+    // the problem of later having to mark those objects if the referent is
+    // still final reachable during processing.
+    volatile oop* const referent_addr = reference_referent_addr(reference);
+    XBarrier::mark_barrier_on_oop_field(referent_addr, true /* finalizable */);
+  }
+
+  // Add reference to discovered list
+  assert(reference_discovered(reference) == NULL, "Already discovered");
+  oop* const list = _discovered_list.addr();
+  reference_set_discovered(reference, *list);
+  *list = reference;
+}
+
+bool XReferenceProcessor::discover_reference(oop reference, ReferenceType type) {
+  if (!RegisterReferences) {
+    // Reference processing disabled
+    return false;
+  }
+
+  log_trace(gc, ref)("Encountered Reference: " PTR_FORMAT " (%s)", p2i(reference), reference_type_name(type));
+
+  // Update statistics
+  _encountered_count.get()[type]++;
+
+  if (!should_discover(reference, type)) {
+    // Not discovered
+    return false;
+  }
+
+  discover(reference, type);
+
+  // Discovered
+  return true;
+}
+
+oop XReferenceProcessor::drop(oop reference, ReferenceType type) {
+  log_trace(gc, ref)("Dropped Reference: " PTR_FORMAT " (%s)", p2i(reference), reference_type_name(type));
+
+  // Keep referent alive
+  keep_alive(reference, type);
+
+  // Unlink and return next in list
+  const oop next = reference_discovered(reference);
+  reference_set_discovered(reference, NULL);
+  return next;
+}
+
+oop* XReferenceProcessor::keep(oop reference, ReferenceType type) {
+  log_trace(gc, ref)("Enqueued Reference: " PTR_FORMAT " (%s)", p2i(reference), reference_type_name(type));
+
+  // Update statistics
+  _enqueued_count.get()[type]++;
+
+  // Make reference inactive
+  make_inactive(reference, type);
+
+  // Return next in list
+  return reference_discovered_addr(reference);
+}
+
+void XReferenceProcessor::work() {
+  // Process discovered references
+  oop* const list = _discovered_list.addr();
+  oop* p = list;
+
+  while (*p != NULL) {
+    const oop reference = *p;
+    const ReferenceType type = reference_type(reference);
+
+    if (should_drop(reference, type)) {
+      *p = drop(reference, type);
+    } else {
+      p = keep(reference, type);
+    }
+  }
+
+  // Prepend discovered references to internal pending list
+  if (*list != NULL) {
+    *p = Atomic::xchg(_pending_list.addr(), *list);
+    if (*p == NULL) {
+      // First to prepend to list, record tail
+      _pending_list_tail = p;
+    }
+
+    // Clear discovered list
+    *list = NULL;
+  }
+}
+
+bool XReferenceProcessor::is_empty() const {
+  XPerWorkerConstIterator<oop> iter(&_discovered_list);
+  for (const oop* list; iter.next(&list);) {
+    if (*list != NULL) {
+      return false;
+    }
+  }
+
+  if (_pending_list.get() != NULL) {
+    return false;
+  }
+
+  return true;
+}
+
+void XReferenceProcessor::reset_statistics() {
+  assert(is_empty(), "Should be empty");
+
+  // Reset encountered
+  XPerWorkerIterator<Counters> iter_encountered(&_encountered_count);
+  for (Counters* counters; iter_encountered.next(&counters);) {
+    for (int i = REF_SOFT; i <= REF_PHANTOM; i++) {
+      (*counters)[i] = 0;
+    }
+  }
+
+  // Reset discovered
+  XPerWorkerIterator<Counters> iter_discovered(&_discovered_count);
+  for (Counters* counters; iter_discovered.next(&counters);) {
+    for (int i = REF_SOFT; i <= REF_PHANTOM; i++) {
+      (*counters)[i] = 0;
+    }
+  }
+
+  // Reset enqueued
+  XPerWorkerIterator<Counters> iter_enqueued(&_enqueued_count);
+  for (Counters* counters; iter_enqueued.next(&counters);) {
+    for (int i = REF_SOFT; i <= REF_PHANTOM; i++) {
+      (*counters)[i] = 0;
+    }
+  }
+}
+
+void XReferenceProcessor::collect_statistics() {
+  Counters encountered = {};
+  Counters discovered = {};
+  Counters enqueued = {};
+
+  // Sum encountered
+  XPerWorkerConstIterator<Counters> iter_encountered(&_encountered_count);
+  for (const Counters* counters; iter_encountered.next(&counters);) {
+    for (int i = REF_SOFT; i <= REF_PHANTOM; i++) {
+      encountered[i] += (*counters)[i];
+    }
+  }
+
+  // Sum discovered
+  XPerWorkerConstIterator<Counters> iter_discovered(&_discovered_count);
+  for (const Counters* counters; iter_discovered.next(&counters);) {
+    for (int i = REF_SOFT; i <= REF_PHANTOM; i++) {
+      discovered[i] += (*counters)[i];
+    }
+  }
+
+  // Sum enqueued
+  XPerWorkerConstIterator<Counters> iter_enqueued(&_enqueued_count);
+  for (const Counters* counters; iter_enqueued.next(&counters);) {
+    for (int i = REF_SOFT; i <= REF_PHANTOM; i++) {
+      enqueued[i] += (*counters)[i];
+    }
+  }
+
+  // Update statistics
+  XStatReferences::set_soft(encountered[REF_SOFT], discovered[REF_SOFT], enqueued[REF_SOFT]);
+  XStatReferences::set_weak(encountered[REF_WEAK], discovered[REF_WEAK], enqueued[REF_WEAK]);
+  XStatReferences::set_final(encountered[REF_FINAL], discovered[REF_FINAL], enqueued[REF_FINAL]);
+  XStatReferences::set_phantom(encountered[REF_PHANTOM], discovered[REF_PHANTOM], enqueued[REF_PHANTOM]);
+
+  // Trace statistics
+  const ReferenceProcessorStats stats(discovered[REF_SOFT],
+                                      discovered[REF_WEAK],
+                                      discovered[REF_FINAL],
+                                      discovered[REF_PHANTOM]);
+  XTracer::tracer()->report_gc_reference_stats(stats);
+}
+
+class XReferenceProcessorTask : public XTask {
+private:
+  XReferenceProcessor* const _reference_processor;
+
+public:
+  XReferenceProcessorTask(XReferenceProcessor* reference_processor) :
+      XTask("XReferenceProcessorTask"),
+      _reference_processor(reference_processor) {}
+
+  virtual void work() {
+    _reference_processor->work();
+  }
+};
+
+void XReferenceProcessor::process_references() {
+  XStatTimer timer(XSubPhaseConcurrentReferencesProcess);
+
+  // Process discovered lists
+  XReferenceProcessorTask task(this);
+  _workers->run(&task);
+
+  // Update SoftReference clock
+  soft_reference_update_clock();
+
+  // Collect, log and trace statistics
+  collect_statistics();
+}
+
+void XReferenceProcessor::enqueue_references() {
+  XStatTimer timer(XSubPhaseConcurrentReferencesEnqueue);
+
+  if (_pending_list.get() == NULL) {
+    // Nothing to enqueue
+    return;
+  }
+
+  {
+    // Heap_lock protects external pending list
+    MonitorLocker ml(Heap_lock);
+
+    // Prepend internal pending list to external pending list
+    *_pending_list_tail = Universe::swap_reference_pending_list(_pending_list.get());
+
+    // Notify ReferenceHandler thread
+    ml.notify_all();
+  }
+
+  // Reset internal pending list
+  _pending_list.set(NULL);
+  _pending_list_tail = _pending_list.addr();
+}
diff --git a/src/hotspot/share/gc/x/xReferenceProcessor.hpp b/src/hotspot/share/gc/x/xReferenceProcessor.hpp
new file mode 100644
index 00000000000..1ff7b14e868
--- /dev/null
+++ b/src/hotspot/share/gc/x/xReferenceProcessor.hpp
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XREFERENCEPROCESSOR_HPP
+#define SHARE_GC_X_XREFERENCEPROCESSOR_HPP
+
+#include "gc/shared/referenceDiscoverer.hpp"
+#include "gc/x/xValue.hpp"
+
+class ReferencePolicy;
+class XWorkers;
+
+class XReferenceProcessor : public ReferenceDiscoverer {
+  friend class XReferenceProcessorTask;
+
+private:
+  static const size_t reference_type_count = REF_PHANTOM + 1;
+  typedef size_t Counters[reference_type_count];
+
+  XWorkers* const      _workers;
+  ReferencePolicy*     _soft_reference_policy;
+  XPerWorker<Counters> _encountered_count;
+  XPerWorker<Counters> _discovered_count;
+  XPerWorker<Counters> _enqueued_count;
+  XPerWorker<oop>      _discovered_list;
+  XContended<oop>      _pending_list;
+  oop*                 _pending_list_tail;
+
+  bool is_inactive(oop reference, oop referent, ReferenceType type) const;
+  bool is_strongly_live(oop referent) const;
+  bool is_softly_live(oop reference, ReferenceType type) const;
+
+  bool should_discover(oop reference, ReferenceType type) const;
+  bool should_drop(oop reference, ReferenceType type) const;
+  void keep_alive(oop reference, ReferenceType type) const;
+  void make_inactive(oop reference, ReferenceType type) const;
+
+  void discover(oop reference, ReferenceType type);
+
+  oop drop(oop reference, ReferenceType type);
+  oop* keep(oop reference, ReferenceType type);
+
+  bool is_empty() const;
+
+  void work();
+  void collect_statistics();
+
+public:
+  XReferenceProcessor(XWorkers* workers);
+
+  void set_soft_reference_policy(bool clear);
+  void reset_statistics();
+
+  virtual bool discover_reference(oop reference, ReferenceType type);
+  void process_references();
+  void enqueue_references();
+};
+
+#endif // SHARE_GC_X_XREFERENCEPROCESSOR_HPP
diff --git a/src/hotspot/share/gc/x/xRelocate.cpp b/src/hotspot/share/gc/x/xRelocate.cpp
new file mode 100644
index 00000000000..e13773242b3
--- /dev/null
+++ b/src/hotspot/share/gc/x/xRelocate.cpp
@@ -0,0 +1,419 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/xAbort.inline.hpp"
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xBarrier.inline.hpp"
+#include "gc/x/xForwarding.inline.hpp"
+#include "gc/x/xHeap.inline.hpp"
+#include "gc/x/xPage.inline.hpp"
+#include "gc/x/xRelocate.hpp"
+#include "gc/x/xRelocationSet.inline.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xTask.hpp"
+#include "gc/x/xThread.inline.hpp"
+#include "gc/x/xWorkers.hpp"
+#include "prims/jvmtiTagMap.hpp"
+#include "runtime/atomic.hpp"
+#include "utilities/debug.hpp"
+
+XRelocate::XRelocate(XWorkers* workers) :
+    _workers(workers) {}
+
+static uintptr_t forwarding_index(XForwarding* forwarding, uintptr_t from_addr) {
+  const uintptr_t from_offset = XAddress::offset(from_addr);
+  return (from_offset - forwarding->start()) >> forwarding->object_alignment_shift();
+}
+
+static uintptr_t forwarding_find(XForwarding* forwarding, uintptr_t from_addr, XForwardingCursor* cursor) {
+  const uintptr_t from_index = forwarding_index(forwarding, from_addr);
+  const XForwardingEntry entry = forwarding->find(from_index, cursor);
+  return entry.populated() ? XAddress::good(entry.to_offset()) : 0;
+}
+
+static uintptr_t forwarding_insert(XForwarding* forwarding, uintptr_t from_addr, uintptr_t to_addr, XForwardingCursor* cursor) {
+  const uintptr_t from_index = forwarding_index(forwarding, from_addr);
+  const uintptr_t to_offset = XAddress::offset(to_addr);
+  const uintptr_t to_offset_final = forwarding->insert(from_index, to_offset, cursor);
+  return XAddress::good(to_offset_final);
+}
+
+static uintptr_t relocate_object_inner(XForwarding* forwarding, uintptr_t from_addr, XForwardingCursor* cursor) {
+  assert(XHeap::heap()->is_object_live(from_addr), "Should be live");
+
+  // Allocate object
+  const size_t size = XUtils::object_size(from_addr);
+  const uintptr_t to_addr = XHeap::heap()->alloc_object_for_relocation(size);
+  if (to_addr == 0) {
+    // Allocation failed
+    return 0;
+  }
+
+  // Copy object
+  XUtils::object_copy_disjoint(from_addr, to_addr, size);
+
+  // Insert forwarding
+  const uintptr_t to_addr_final = forwarding_insert(forwarding, from_addr, to_addr, cursor);
+  if (to_addr_final != to_addr) {
+    // Already relocated, try undo allocation
+    XHeap::heap()->undo_alloc_object_for_relocation(to_addr, size);
+  }
+
+  return to_addr_final;
+}
+
+uintptr_t XRelocate::relocate_object(XForwarding* forwarding, uintptr_t from_addr) const {
+  XForwardingCursor cursor;
+
+  // Lookup forwarding
+  uintptr_t to_addr = forwarding_find(forwarding, from_addr, &cursor);
+  if (to_addr != 0) {
+    // Already relocated
+    return to_addr;
+  }
+
+  // Relocate object
+  if (forwarding->retain_page()) {
+    to_addr = relocate_object_inner(forwarding, from_addr, &cursor);
+    forwarding->release_page();
+
+    if (to_addr != 0) {
+      // Success
+      return to_addr;
+    }
+
+    // Failed to relocate object. Wait for a worker thread to complete
+    // relocation of this page, and then forward the object. If the GC
+    // aborts the relocation phase before the page has been relocated,
+    // then wait return false and we just forward the object in-place.
+    if (!forwarding->wait_page_released()) {
+      // Forward object in-place
+      return forwarding_insert(forwarding, from_addr, from_addr, &cursor);
+    }
+  }
+
+  // Forward object
+  return forward_object(forwarding, from_addr);
+}
+
+uintptr_t XRelocate::forward_object(XForwarding* forwarding, uintptr_t from_addr) const {
+  XForwardingCursor cursor;
+  const uintptr_t to_addr = forwarding_find(forwarding, from_addr, &cursor);
+  assert(to_addr != 0, "Should be forwarded");
+  return to_addr;
+}
+
+static XPage* alloc_page(const XForwarding* forwarding) {
+  if (ZStressRelocateInPlace) {
+    // Simulate failure to allocate a new page. This will
+    // cause the page being relocated to be relocated in-place.
+    return NULL;
+  }
+
+  XAllocationFlags flags;
+  flags.set_non_blocking();
+  flags.set_worker_relocation();
+  return XHeap::heap()->alloc_page(forwarding->type(), forwarding->size(), flags);
+}
+
+static void free_page(XPage* page) {
+  XHeap::heap()->free_page(page, true /* reclaimed */);
+}
+
+static bool should_free_target_page(XPage* page) {
+  // Free target page if it is empty. We can end up with an empty target
+  // page if we allocated a new target page, and then lost the race to
+  // relocate the remaining objects, leaving the target page empty when
+  // relocation completed.
+  return page != NULL && page->top() == page->start();
+}
+
+class XRelocateSmallAllocator {
+private:
+  volatile size_t _in_place_count;
+
+public:
+  XRelocateSmallAllocator() :
+      _in_place_count(0) {}
+
+  XPage* alloc_target_page(XForwarding* forwarding, XPage* target) {
+    XPage* const page = alloc_page(forwarding);
+    if (page == NULL) {
+      Atomic::inc(&_in_place_count);
+    }
+
+    return page;
+  }
+
+  void share_target_page(XPage* page) {
+    // Does nothing
+  }
+
+  void free_target_page(XPage* page) {
+    if (should_free_target_page(page)) {
+      free_page(page);
+    }
+  }
+
+  void free_relocated_page(XPage* page) {
+    free_page(page);
+  }
+
+  uintptr_t alloc_object(XPage* page, size_t size) const {
+    return (page != NULL) ? page->alloc_object(size) : 0;
+  }
+
+  void undo_alloc_object(XPage* page, uintptr_t addr, size_t size) const {
+    page->undo_alloc_object(addr, size);
+  }
+
+  const size_t in_place_count() const {
+    return _in_place_count;
+  }
+};
+
+class XRelocateMediumAllocator {
+private:
+  XConditionLock      _lock;
+  XPage*              _shared;
+  bool                _in_place;
+  volatile size_t     _in_place_count;
+
+public:
+  XRelocateMediumAllocator() :
+      _lock(),
+      _shared(NULL),
+      _in_place(false),
+      _in_place_count(0) {}
+
+  ~XRelocateMediumAllocator() {
+    if (should_free_target_page(_shared)) {
+      free_page(_shared);
+    }
+  }
+
+  XPage* alloc_target_page(XForwarding* forwarding, XPage* target) {
+    XLocker<XConditionLock> locker(&_lock);
+
+    // Wait for any ongoing in-place relocation to complete
+    while (_in_place) {
+      _lock.wait();
+    }
+
+    // Allocate a new page only if the shared page is the same as the
+    // current target page. The shared page will be different from the
+    // current target page if another thread shared a page, or allocated
+    // a new page.
+    if (_shared == target) {
+      _shared = alloc_page(forwarding);
+      if (_shared == NULL) {
+        Atomic::inc(&_in_place_count);
+        _in_place = true;
+      }
+    }
+
+    return _shared;
+  }
+
+  void share_target_page(XPage* page) {
+    XLocker<XConditionLock> locker(&_lock);
+
+    assert(_in_place, "Invalid state");
+    assert(_shared == NULL, "Invalid state");
+    assert(page != NULL, "Invalid page");
+
+    _shared = page;
+    _in_place = false;
+
+    _lock.notify_all();
+  }
+
+  void free_target_page(XPage* page) {
+    // Does nothing
+  }
+
+  void free_relocated_page(XPage* page) {
+    free_page(page);
+  }
+
+  uintptr_t alloc_object(XPage* page, size_t size) const {
+    return (page != NULL) ? page->alloc_object_atomic(size) : 0;
+  }
+
+  void undo_alloc_object(XPage* page, uintptr_t addr, size_t size) const {
+    page->undo_alloc_object_atomic(addr, size);
+  }
+
+  const size_t in_place_count() const {
+    return _in_place_count;
+  }
+};
+
+template <typename Allocator>
+class XRelocateClosure : public ObjectClosure {
+private:
+  Allocator* const _allocator;
+  XForwarding*     _forwarding;
+  XPage*           _target;
+
+  bool relocate_object(uintptr_t from_addr) const {
+    XForwardingCursor cursor;
+
+    // Lookup forwarding
+    if (forwarding_find(_forwarding, from_addr, &cursor) != 0) {
+      // Already relocated
+      return true;
+    }
+
+    // Allocate object
+    const size_t size = XUtils::object_size(from_addr);
+    const uintptr_t to_addr = _allocator->alloc_object(_target, size);
+    if (to_addr == 0) {
+      // Allocation failed
+      return false;
+    }
+
+    // Copy object. Use conjoint copying if we are relocating
+    // in-place and the new object overlapps with the old object.
+    if (_forwarding->in_place() && to_addr + size > from_addr) {
+      XUtils::object_copy_conjoint(from_addr, to_addr, size);
+    } else {
+      XUtils::object_copy_disjoint(from_addr, to_addr, size);
+    }
+
+    // Insert forwarding
+    if (forwarding_insert(_forwarding, from_addr, to_addr, &cursor) != to_addr) {
+      // Already relocated, undo allocation
+      _allocator->undo_alloc_object(_target, to_addr, size);
+    }
+
+    return true;
+  }
+
+  virtual void do_object(oop obj) {
+    const uintptr_t addr = XOop::to_address(obj);
+    assert(XHeap::heap()->is_object_live(addr), "Should be live");
+
+    while (!relocate_object(addr)) {
+      // Allocate a new target page, or if that fails, use the page being
+      // relocated as the new target, which will cause it to be relocated
+      // in-place.
+      _target = _allocator->alloc_target_page(_forwarding, _target);
+      if (_target != NULL) {
+        continue;
+      }
+
+      // Claim the page being relocated to block other threads from accessing
+      // it, or its forwarding table, until it has been released (relocation
+      // completed).
+      _target = _forwarding->claim_page();
+      _target->reset_for_in_place_relocation();
+      _forwarding->set_in_place();
+    }
+  }
+
+public:
+  XRelocateClosure(Allocator* allocator) :
+      _allocator(allocator),
+      _forwarding(NULL),
+      _target(NULL) {}
+
+  ~XRelocateClosure() {
+    _allocator->free_target_page(_target);
+  }
+
+  void do_forwarding(XForwarding* forwarding) {
+    _forwarding = forwarding;
+
+    // Check if we should abort
+    if (XAbort::should_abort()) {
+      _forwarding->abort_page();
+      return;
+    }
+
+    // Relocate objects
+    _forwarding->object_iterate(this);
+
+    // Verify
+    if (ZVerifyForwarding) {
+      _forwarding->verify();
+    }
+
+    // Release relocated page
+    _forwarding->release_page();
+
+    if (_forwarding->in_place()) {
+      // The relocated page has been relocated in-place and should not
+      // be freed. Keep it as target page until it is full, and offer to
+      // share it with other worker threads.
+      _allocator->share_target_page(_target);
+    } else {
+      // Detach and free relocated page
+      XPage* const page = _forwarding->detach_page();
+      _allocator->free_relocated_page(page);
+    }
+  }
+};
+
+class XRelocateTask : public XTask {
+private:
+  XRelocationSetParallelIterator _iter;
+  XRelocateSmallAllocator        _small_allocator;
+  XRelocateMediumAllocator       _medium_allocator;
+
+  static bool is_small(XForwarding* forwarding) {
+    return forwarding->type() == XPageTypeSmall;
+  }
+
+public:
+  XRelocateTask(XRelocationSet* relocation_set) :
+      XTask("XRelocateTask"),
+      _iter(relocation_set),
+      _small_allocator(),
+      _medium_allocator() {}
+
+  ~XRelocateTask() {
+    XStatRelocation::set_at_relocate_end(_small_allocator.in_place_count(),
+                                         _medium_allocator.in_place_count());
+  }
+
+  virtual void work() {
+    XRelocateClosure<XRelocateSmallAllocator> small(&_small_allocator);
+    XRelocateClosure<XRelocateMediumAllocator> medium(&_medium_allocator);
+
+    for (XForwarding* forwarding; _iter.next(&forwarding);) {
+      if (is_small(forwarding)) {
+        small.do_forwarding(forwarding);
+      } else {
+        medium.do_forwarding(forwarding);
+      }
+    }
+  }
+};
+
+void XRelocate::relocate(XRelocationSet* relocation_set) {
+  XRelocateTask task(relocation_set);
+  _workers->run(&task);
+}
diff --git a/src/hotspot/share/gc/x/xRelocate.hpp b/src/hotspot/share/gc/x/xRelocate.hpp
new file mode 100644
index 00000000000..46ab39240f6
--- /dev/null
+++ b/src/hotspot/share/gc/x/xRelocate.hpp
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XRELOCATE_HPP
+#define SHARE_GC_X_XRELOCATE_HPP
+
+#include "gc/x/xRelocationSet.hpp"
+
+class XForwarding;
+class XWorkers;
+
+class XRelocate {
+  friend class XRelocateTask;
+
+private:
+  XWorkers* const _workers;
+
+  void work(XRelocationSetParallelIterator* iter);
+
+public:
+  XRelocate(XWorkers* workers);
+
+  uintptr_t relocate_object(XForwarding* forwarding, uintptr_t from_addr) const;
+  uintptr_t forward_object(XForwarding* forwarding, uintptr_t from_addr) const;
+
+  void relocate(XRelocationSet* relocation_set);
+};
+
+#endif // SHARE_GC_X_XRELOCATE_HPP
diff --git a/src/hotspot/share/gc/x/xRelocationSet.cpp b/src/hotspot/share/gc/x/xRelocationSet.cpp
new file mode 100644
index 00000000000..ff3ac6976cd
--- /dev/null
+++ b/src/hotspot/share/gc/x/xRelocationSet.cpp
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xArray.inline.hpp"
+#include "gc/x/xForwarding.inline.hpp"
+#include "gc/x/xForwardingAllocator.inline.hpp"
+#include "gc/x/xRelocationSet.inline.hpp"
+#include "gc/x/xRelocationSetSelector.inline.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xTask.hpp"
+#include "gc/x/xWorkers.hpp"
+#include "runtime/atomic.hpp"
+#include "utilities/debug.hpp"
+
+class XRelocationSetInstallTask : public XTask {
+private:
+  XForwardingAllocator* const    _allocator;
+  XForwarding**                  _forwardings;
+  const size_t                   _nforwardings;
+  XArrayParallelIterator<XPage*> _small_iter;
+  XArrayParallelIterator<XPage*> _medium_iter;
+  volatile size_t                _small_next;
+  volatile size_t                _medium_next;
+
+  void install(XForwarding* forwarding, volatile size_t* next) {
+    const size_t index = Atomic::fetch_and_add(next, 1u);
+    assert(index < _nforwardings, "Invalid index");
+    _forwardings[index] = forwarding;
+  }
+
+  void install_small(XForwarding* forwarding) {
+    install(forwarding, &_small_next);
+  }
+
+  void install_medium(XForwarding* forwarding) {
+    install(forwarding, &_medium_next);
+  }
+
+public:
+  XRelocationSetInstallTask(XForwardingAllocator* allocator, const XRelocationSetSelector* selector) :
+      XTask("XRelocationSetInstallTask"),
+      _allocator(allocator),
+      _forwardings(NULL),
+      _nforwardings(selector->small()->length() + selector->medium()->length()),
+      _small_iter(selector->small()),
+      _medium_iter(selector->medium()),
+      _small_next(selector->medium()->length()),
+      _medium_next(0) {
+
+    // Reset the allocator to have room for the relocation
+    // set, all forwardings, and all forwarding entries.
+    const size_t relocation_set_size = _nforwardings * sizeof(XForwarding*);
+    const size_t forwardings_size = _nforwardings * sizeof(XForwarding);
+    const size_t forwarding_entries_size = selector->forwarding_entries() * sizeof(XForwardingEntry);
+    _allocator->reset(relocation_set_size + forwardings_size + forwarding_entries_size);
+
+    // Allocate relocation set
+    _forwardings = new (_allocator->alloc(relocation_set_size)) XForwarding*[_nforwardings];
+  }
+
+  ~XRelocationSetInstallTask() {
+    assert(_allocator->is_full(), "Should be full");
+  }
+
+  virtual void work() {
+    // Allocate and install forwardings for small pages
+    for (XPage* page; _small_iter.next(&page);) {
+      XForwarding* const forwarding = XForwarding::alloc(_allocator, page);
+      install_small(forwarding);
+    }
+
+    // Allocate and install forwardings for medium pages
+    for (XPage* page; _medium_iter.next(&page);) {
+      XForwarding* const forwarding = XForwarding::alloc(_allocator, page);
+      install_medium(forwarding);
+    }
+  }
+
+  XForwarding** forwardings() const {
+    return _forwardings;
+  }
+
+  size_t nforwardings() const {
+    return _nforwardings;
+  }
+};
+
+XRelocationSet::XRelocationSet(XWorkers* workers) :
+    _workers(workers),
+    _allocator(),
+    _forwardings(NULL),
+    _nforwardings(0) {}
+
+void XRelocationSet::install(const XRelocationSetSelector* selector) {
+  // Install relocation set
+  XRelocationSetInstallTask task(&_allocator, selector);
+  _workers->run(&task);
+
+  _forwardings = task.forwardings();
+  _nforwardings = task.nforwardings();
+
+  // Update statistics
+  XStatRelocation::set_at_install_relocation_set(_allocator.size());
+}
+
+void XRelocationSet::reset() {
+  // Destroy forwardings
+  XRelocationSetIterator iter(this);
+  for (XForwarding* forwarding; iter.next(&forwarding);) {
+    forwarding->~XForwarding();
+  }
+
+  _nforwardings = 0;
+}
diff --git a/src/hotspot/share/gc/x/xRelocationSet.hpp b/src/hotspot/share/gc/x/xRelocationSet.hpp
new file mode 100644
index 00000000000..bbbb3770516
--- /dev/null
+++ b/src/hotspot/share/gc/x/xRelocationSet.hpp
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XRELOCATIONSET_HPP
+#define SHARE_GC_X_XRELOCATIONSET_HPP
+
+#include "gc/x/xArray.hpp"
+#include "gc/x/xForwardingAllocator.hpp"
+
+class XForwarding;
+class XRelocationSetSelector;
+class XWorkers;
+
+class XRelocationSet {
+  template <bool> friend class XRelocationSetIteratorImpl;
+
+private:
+  XWorkers*            _workers;
+  XForwardingAllocator _allocator;
+  XForwarding**        _forwardings;
+  size_t               _nforwardings;
+
+public:
+  XRelocationSet(XWorkers* workers);
+
+  void install(const XRelocationSetSelector* selector);
+  void reset();
+};
+
+template <bool Parallel>
+class XRelocationSetIteratorImpl : public XArrayIteratorImpl<XForwarding*, Parallel> {
+public:
+  XRelocationSetIteratorImpl(XRelocationSet* relocation_set);
+};
+
+using XRelocationSetIterator = XRelocationSetIteratorImpl<false /* Parallel */>;
+using XRelocationSetParallelIterator = XRelocationSetIteratorImpl<true /* Parallel */>;
+
+#endif // SHARE_GC_X_XRELOCATIONSET_HPP
diff --git a/src/hotspot/share/gc/x/xRelocationSet.inline.hpp b/src/hotspot/share/gc/x/xRelocationSet.inline.hpp
new file mode 100644
index 00000000000..3b76fbce46a
--- /dev/null
+++ b/src/hotspot/share/gc/x/xRelocationSet.inline.hpp
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XRELOCATIONSET_INLINE_HPP
+#define SHARE_GC_X_XRELOCATIONSET_INLINE_HPP
+
+#include "gc/x/xRelocationSet.hpp"
+
+#include "gc/x/xArray.inline.hpp"
+
+template <bool Parallel>
+inline XRelocationSetIteratorImpl<Parallel>::XRelocationSetIteratorImpl(XRelocationSet* relocation_set) :
+    XArrayIteratorImpl<XForwarding*, Parallel>(relocation_set->_forwardings, relocation_set->_nforwardings) {}
+
+#endif // SHARE_GC_X_XRELOCATIONSET_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xRelocationSetSelector.cpp b/src/hotspot/share/gc/x/xRelocationSetSelector.cpp
new file mode 100644
index 00000000000..b009443d395
--- /dev/null
+++ b/src/hotspot/share/gc/x/xRelocationSetSelector.cpp
@@ -0,0 +1,213 @@
+/*
+ * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/xArray.inline.hpp"
+#include "gc/x/xForwarding.inline.hpp"
+#include "gc/x/xPage.inline.hpp"
+#include "gc/x/xRelocationSetSelector.inline.hpp"
+#include "jfr/jfrEvents.hpp"
+#include "logging/log.hpp"
+#include "runtime/globals.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/powerOfTwo.hpp"
+
+XRelocationSetSelectorGroupStats::XRelocationSetSelectorGroupStats() :
+    _npages_candidates(0),
+    _total(0),
+    _live(0),
+    _empty(0),
+    _npages_selected(0),
+    _relocate(0) {}
+
+XRelocationSetSelectorGroup::XRelocationSetSelectorGroup(const char* name,
+                                                         uint8_t page_type,
+                                                         size_t page_size,
+                                                         size_t object_size_limit) :
+    _name(name),
+    _page_type(page_type),
+    _page_size(page_size),
+    _object_size_limit(object_size_limit),
+    _fragmentation_limit(page_size * (ZFragmentationLimit / 100)),
+    _live_pages(),
+    _forwarding_entries(0),
+    _stats() {}
+
+bool XRelocationSetSelectorGroup::is_disabled() {
+  // Medium pages are disabled when their page size is zero
+  return _page_type == XPageTypeMedium && _page_size == 0;
+}
+
+bool XRelocationSetSelectorGroup::is_selectable() {
+  // Large pages are not selectable
+  return _page_type != XPageTypeLarge;
+}
+
+void XRelocationSetSelectorGroup::semi_sort() {
+  // Semi-sort live pages by number of live bytes in ascending order
+  const size_t npartitions_shift = 11;
+  const size_t npartitions = (size_t)1 << npartitions_shift;
+  const size_t partition_size = _page_size >> npartitions_shift;
+  const size_t partition_size_shift = exact_log2(partition_size);
+
+  // Partition slots/fingers
+  int partitions[npartitions] = { /* zero initialize */ };
+
+  // Calculate partition slots
+  XArrayIterator<XPage*> iter1(&_live_pages);
+  for (XPage* page; iter1.next(&page);) {
+    const size_t index = page->live_bytes() >> partition_size_shift;
+    partitions[index]++;
+  }
+
+  // Calculate partition fingers
+  int finger = 0;
+  for (size_t i = 0; i < npartitions; i++) {
+    const int slots = partitions[i];
+    partitions[i] = finger;
+    finger += slots;
+  }
+
+  // Allocate destination array
+  const int npages = _live_pages.length();
+  XArray<XPage*> sorted_live_pages(npages, npages, NULL);
+
+  // Sort pages into partitions
+  XArrayIterator<XPage*> iter2(&_live_pages);
+  for (XPage* page; iter2.next(&page);) {
+    const size_t index = page->live_bytes() >> partition_size_shift;
+    const int finger = partitions[index]++;
+    assert(sorted_live_pages.at(finger) == NULL, "Invalid finger");
+    sorted_live_pages.at_put(finger, page);
+  }
+
+  _live_pages.swap(&sorted_live_pages);
+}
+
+void XRelocationSetSelectorGroup::select_inner() {
+  // Calculate the number of pages to relocate by successively including pages in
+  // a candidate relocation set and calculate the maximum space requirement for
+  // their live objects.
+  const int npages = _live_pages.length();
+  int selected_from = 0;
+  int selected_to = 0;
+  size_t npages_selected = 0;
+  size_t selected_live_bytes = 0;
+  size_t selected_forwarding_entries = 0;
+  size_t from_live_bytes = 0;
+  size_t from_forwarding_entries = 0;
+
+  semi_sort();
+
+  for (int from = 1; from <= npages; from++) {
+    // Add page to the candidate relocation set
+    XPage* const page = _live_pages.at(from - 1);
+    from_live_bytes += page->live_bytes();
+    from_forwarding_entries += XForwarding::nentries(page);
+
+    // Calculate the maximum number of pages needed by the candidate relocation set.
+    // By subtracting the object size limit from the pages size we get the maximum
+    // number of pages that the relocation set is guaranteed to fit in, regardless
+    // of in which order the objects are relocated.
+    const int to = ceil((double)(from_live_bytes) / (double)(_page_size - _object_size_limit));
+
+    // Calculate the relative difference in reclaimable space compared to our
+    // currently selected final relocation set. If this number is larger than the
+    // acceptable fragmentation limit, then the current candidate relocation set
+    // becomes our new final relocation set.
+    const int diff_from = from - selected_from;
+    const int diff_to = to - selected_to;
+    const double diff_reclaimable = 100 - percent_of(diff_to, diff_from);
+    if (diff_reclaimable > ZFragmentationLimit) {
+      selected_from = from;
+      selected_to = to;
+      selected_live_bytes = from_live_bytes;
+      npages_selected += 1;
+      selected_forwarding_entries = from_forwarding_entries;
+    }
+
+    log_trace(gc, reloc)("Candidate Relocation Set (%s Pages): %d->%d, "
+                         "%.1f%% relative defragmentation, " SIZE_FORMAT " forwarding entries, %s",
+                         _name, from, to, diff_reclaimable, from_forwarding_entries,
+                         (selected_from == from) ? "Selected" : "Rejected");
+  }
+
+  // Finalize selection
+  _live_pages.trunc_to(selected_from);
+  _forwarding_entries = selected_forwarding_entries;
+
+  // Update statistics
+  _stats._relocate = selected_live_bytes;
+  _stats._npages_selected = npages_selected;
+
+  log_trace(gc, reloc)("Relocation Set (%s Pages): %d->%d, %d skipped, " SIZE_FORMAT " forwarding entries",
+                       _name, selected_from, selected_to, npages - selected_from, selected_forwarding_entries);
+}
+
+void XRelocationSetSelectorGroup::select() {
+  if (is_disabled()) {
+    return;
+  }
+
+  EventZRelocationSetGroup event;
+
+  if (is_selectable()) {
+    select_inner();
+  }
+
+  // Send event
+  event.commit(_page_type, _stats.npages_candidates(), _stats.total(), _stats.empty(), _stats.npages_selected(), _stats.relocate());
+}
+
+XRelocationSetSelector::XRelocationSetSelector() :
+    _small("Small", XPageTypeSmall, XPageSizeSmall, XObjectSizeLimitSmall),
+    _medium("Medium", XPageTypeMedium, XPageSizeMedium, XObjectSizeLimitMedium),
+    _large("Large", XPageTypeLarge, 0 /* page_size */, 0 /* object_size_limit */),
+    _empty_pages() {}
+
+void XRelocationSetSelector::select() {
+  // Select pages to relocate. The resulting relocation set will be
+  // sorted such that medium pages comes first, followed by small
+  // pages. Pages within each page group will be semi-sorted by live
+  // bytes in ascending order. Relocating pages in this order allows
+  // us to start reclaiming memory more quickly.
+
+  EventZRelocationSet event;
+
+  // Select pages from each group
+  _large.select();
+  _medium.select();
+  _small.select();
+
+  // Send event
+  event.commit(total(), empty(), relocate());
+}
+
+XRelocationSetSelectorStats XRelocationSetSelector::stats() const {
+  XRelocationSetSelectorStats stats;
+  stats._small = _small.stats();
+  stats._medium = _medium.stats();
+  stats._large = _large.stats();
+  return stats;
+}
diff --git a/src/hotspot/share/gc/x/xRelocationSetSelector.hpp b/src/hotspot/share/gc/x/xRelocationSetSelector.hpp
new file mode 100644
index 00000000000..75e40eeea8c
--- /dev/null
+++ b/src/hotspot/share/gc/x/xRelocationSetSelector.hpp
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XRELOCATIONSETSELECTOR_HPP
+#define SHARE_GC_X_XRELOCATIONSETSELECTOR_HPP
+
+#include "gc/x/xArray.hpp"
+#include "memory/allocation.hpp"
+
+class XPage;
+
+class XRelocationSetSelectorGroupStats {
+  friend class XRelocationSetSelectorGroup;
+
+private:
+  // Candidate set
+  size_t _npages_candidates;
+  size_t _total;
+  size_t _live;
+  size_t _empty;
+
+  // Selected set
+  size_t _npages_selected;
+  size_t _relocate;
+
+public:
+  XRelocationSetSelectorGroupStats();
+
+  size_t npages_candidates() const;
+  size_t total() const;
+  size_t live() const;
+  size_t empty() const;
+
+  size_t npages_selected() const;
+  size_t relocate() const;
+};
+
+class XRelocationSetSelectorStats {
+  friend class XRelocationSetSelector;
+
+private:
+  XRelocationSetSelectorGroupStats _small;
+  XRelocationSetSelectorGroupStats _medium;
+  XRelocationSetSelectorGroupStats _large;
+
+public:
+  const XRelocationSetSelectorGroupStats& small() const;
+  const XRelocationSetSelectorGroupStats& medium() const;
+  const XRelocationSetSelectorGroupStats& large() const;
+};
+
+class XRelocationSetSelectorGroup {
+private:
+  const char* const                _name;
+  const uint8_t                    _page_type;
+  const size_t                     _page_size;
+  const size_t                     _object_size_limit;
+  const size_t                     _fragmentation_limit;
+  XArray<XPage*>                   _live_pages;
+  size_t                           _forwarding_entries;
+  XRelocationSetSelectorGroupStats _stats;
+
+  bool is_disabled();
+  bool is_selectable();
+  void semi_sort();
+  void select_inner();
+
+public:
+  XRelocationSetSelectorGroup(const char* name,
+                              uint8_t page_type,
+                              size_t page_size,
+                              size_t object_size_limit);
+
+  void register_live_page(XPage* page);
+  void register_empty_page(XPage* page);
+  void select();
+
+  const XArray<XPage*>* selected() const;
+  size_t forwarding_entries() const;
+
+  const XRelocationSetSelectorGroupStats& stats() const;
+};
+
+class XRelocationSetSelector : public StackObj {
+private:
+  XRelocationSetSelectorGroup _small;
+  XRelocationSetSelectorGroup _medium;
+  XRelocationSetSelectorGroup _large;
+  XArray<XPage*>              _empty_pages;
+
+  size_t total() const;
+  size_t empty() const;
+  size_t relocate() const;
+
+public:
+  XRelocationSetSelector();
+
+  void register_live_page(XPage* page);
+  void register_empty_page(XPage* page);
+
+  bool should_free_empty_pages(int bulk) const;
+  const XArray<XPage*>* empty_pages() const;
+  void clear_empty_pages();
+
+  void select();
+
+  const XArray<XPage*>* small() const;
+  const XArray<XPage*>* medium() const;
+  size_t forwarding_entries() const;
+
+  XRelocationSetSelectorStats stats() const;
+};
+
+#endif // SHARE_GC_X_XRELOCATIONSETSELECTOR_HPP
diff --git a/src/hotspot/share/gc/x/xRelocationSetSelector.inline.hpp b/src/hotspot/share/gc/x/xRelocationSetSelector.inline.hpp
new file mode 100644
index 00000000000..25e0ede835d
--- /dev/null
+++ b/src/hotspot/share/gc/x/xRelocationSetSelector.inline.hpp
@@ -0,0 +1,165 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XRELOCATIONSETSELECTOR_INLINE_HPP
+#define SHARE_GC_X_XRELOCATIONSETSELECTOR_INLINE_HPP
+
+#include "gc/x/xRelocationSetSelector.hpp"
+
+#include "gc/x/xArray.inline.hpp"
+#include "gc/x/xPage.inline.hpp"
+
+inline size_t XRelocationSetSelectorGroupStats::npages_candidates() const {
+  return _npages_candidates;
+}
+
+inline size_t XRelocationSetSelectorGroupStats::total() const {
+  return _total;
+}
+
+inline size_t XRelocationSetSelectorGroupStats::live() const {
+  return _live;
+}
+
+inline size_t XRelocationSetSelectorGroupStats::empty() const {
+  return _empty;
+}
+
+inline size_t XRelocationSetSelectorGroupStats::npages_selected() const {
+  return _npages_selected;
+}
+
+inline size_t XRelocationSetSelectorGroupStats::relocate() const {
+  return _relocate;
+}
+
+inline const XRelocationSetSelectorGroupStats& XRelocationSetSelectorStats::small() const {
+  return _small;
+}
+
+inline const XRelocationSetSelectorGroupStats& XRelocationSetSelectorStats::medium() const {
+  return _medium;
+}
+
+inline const XRelocationSetSelectorGroupStats& XRelocationSetSelectorStats::large() const {
+  return _large;
+}
+
+inline void XRelocationSetSelectorGroup::register_live_page(XPage* page) {
+  const uint8_t type = page->type();
+  const size_t size = page->size();
+  const size_t live = page->live_bytes();
+  const size_t garbage = size - live;
+
+  if (garbage > _fragmentation_limit) {
+    _live_pages.append(page);
+  }
+
+  _stats._npages_candidates++;
+  _stats._total += size;
+  _stats._live += live;
+}
+
+inline void XRelocationSetSelectorGroup::register_empty_page(XPage* page) {
+  const size_t size = page->size();
+
+  _stats._npages_candidates++;
+  _stats._total += size;
+  _stats._empty += size;
+}
+
+inline const XArray<XPage*>* XRelocationSetSelectorGroup::selected() const {
+  return &_live_pages;
+}
+
+inline size_t XRelocationSetSelectorGroup::forwarding_entries() const {
+  return _forwarding_entries;
+}
+
+inline const XRelocationSetSelectorGroupStats& XRelocationSetSelectorGroup::stats() const {
+  return _stats;
+}
+
+inline void XRelocationSetSelector::register_live_page(XPage* page) {
+  const uint8_t type = page->type();
+
+  if (type == XPageTypeSmall) {
+    _small.register_live_page(page);
+  } else if (type == XPageTypeMedium) {
+    _medium.register_live_page(page);
+  } else {
+    _large.register_live_page(page);
+  }
+}
+
+inline void XRelocationSetSelector::register_empty_page(XPage* page) {
+  const uint8_t type = page->type();
+
+  if (type == XPageTypeSmall) {
+    _small.register_empty_page(page);
+  } else if (type == XPageTypeMedium) {
+    _medium.register_empty_page(page);
+  } else {
+    _large.register_empty_page(page);
+  }
+
+  _empty_pages.append(page);
+}
+
+inline bool XRelocationSetSelector::should_free_empty_pages(int bulk) const {
+  return _empty_pages.length() >= bulk && _empty_pages.is_nonempty();
+}
+
+inline const XArray<XPage*>* XRelocationSetSelector::empty_pages() const {
+  return &_empty_pages;
+}
+
+inline void XRelocationSetSelector::clear_empty_pages() {
+  return _empty_pages.clear();
+}
+
+inline size_t XRelocationSetSelector::total() const {
+  return _small.stats().total() + _medium.stats().total() + _large.stats().total();
+}
+
+inline size_t XRelocationSetSelector::empty() const {
+  return _small.stats().empty() + _medium.stats().empty() + _large.stats().empty();
+}
+
+inline size_t XRelocationSetSelector::relocate() const {
+  return _small.stats().relocate() + _medium.stats().relocate() + _large.stats().relocate();
+}
+
+inline const XArray<XPage*>* XRelocationSetSelector::small() const {
+  return _small.selected();
+}
+
+inline const XArray<XPage*>* XRelocationSetSelector::medium() const {
+  return _medium.selected();
+}
+
+inline size_t XRelocationSetSelector::forwarding_entries() const {
+  return _small.forwarding_entries() + _medium.forwarding_entries();
+}
+
+#endif // SHARE_GC_X_XRELOCATIONSETSELECTOR_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xResurrection.cpp b/src/hotspot/share/gc/x/xResurrection.cpp
new file mode 100644
index 00000000000..486f1f8db82
--- /dev/null
+++ b/src/hotspot/share/gc/x/xResurrection.cpp
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xResurrection.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/safepoint.hpp"
+#include "utilities/debug.hpp"
+
+volatile bool XResurrection::_blocked = false;
+
+void XResurrection::block() {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+  _blocked = true;
+}
+
+void XResurrection::unblock() {
+  // No need for anything stronger than a relaxed store here.
+  // The preceding handshake makes sure that all non-strong
+  // oops have already been healed at this point.
+  Atomic::store(&_blocked, false);
+}
diff --git a/src/hotspot/share/gc/x/xResurrection.hpp b/src/hotspot/share/gc/x/xResurrection.hpp
new file mode 100644
index 00000000000..d6ce9820e02
--- /dev/null
+++ b/src/hotspot/share/gc/x/xResurrection.hpp
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2016, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XRESURRECTION_HPP
+#define SHARE_GC_X_XRESURRECTION_HPP
+
+#include "memory/allStatic.hpp"
+
+class XResurrection : public AllStatic {
+private:
+  static volatile bool _blocked;
+
+public:
+  static bool is_blocked();
+  static void block();
+  static void unblock();
+};
+
+#endif // SHARE_GC_X_XRESURRECTION_HPP
diff --git a/src/hotspot/share/gc/x/xResurrection.inline.hpp b/src/hotspot/share/gc/x/xResurrection.inline.hpp
new file mode 100644
index 00000000000..af1993945cc
--- /dev/null
+++ b/src/hotspot/share/gc/x/xResurrection.inline.hpp
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XRESURRECTION_INLINE_HPP
+#define SHARE_GC_X_XRESURRECTION_INLINE_HPP
+
+#include "gc/x/xResurrection.hpp"
+
+#include "runtime/atomic.hpp"
+
+inline bool XResurrection::is_blocked() {
+  return Atomic::load(&_blocked);
+}
+
+#endif // SHARE_GC_X_XRESURRECTION_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xRootsIterator.cpp b/src/hotspot/share/gc/x/xRootsIterator.cpp
new file mode 100644
index 00000000000..1fa44bc219b
--- /dev/null
+++ b/src/hotspot/share/gc/x/xRootsIterator.cpp
@@ -0,0 +1,142 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "classfile/classLoaderDataGraph.hpp"
+#include "gc/shared/oopStorageSetParState.inline.hpp"
+#include "gc/x/xNMethod.hpp"
+#include "gc/x/xNMethodTable.hpp"
+#include "gc/x/xRootsIterator.hpp"
+#include "gc/x/xStat.hpp"
+#include "memory/resourceArea.hpp"
+#include "prims/jvmtiTagMap.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/safepoint.hpp"
+#include "utilities/debug.hpp"
+
+static const XStatSubPhase XSubPhaseConcurrentRootsOopStorageSet("Concurrent Roots OopStorageSet");
+static const XStatSubPhase XSubPhaseConcurrentRootsClassLoaderDataGraph("Concurrent Roots ClassLoaderDataGraph");
+static const XStatSubPhase XSubPhaseConcurrentRootsJavaThreads("Concurrent Roots JavaThreads");
+static const XStatSubPhase XSubPhaseConcurrentRootsCodeCache("Concurrent Roots CodeCache");
+static const XStatSubPhase XSubPhaseConcurrentWeakRootsOopStorageSet("Concurrent Weak Roots OopStorageSet");
+
+template <typename Iterator>
+template <typename ClosureType>
+void XParallelApply<Iterator>::apply(ClosureType* cl) {
+  if (!Atomic::load(&_completed)) {
+    _iter.apply(cl);
+    if (!Atomic::load(&_completed)) {
+      Atomic::store(&_completed, true);
+    }
+  }
+}
+
+XStrongOopStorageSetIterator::XStrongOopStorageSetIterator() :
+    _iter() {}
+
+void XStrongOopStorageSetIterator::apply(OopClosure* cl) {
+  XStatTimer timer(XSubPhaseConcurrentRootsOopStorageSet);
+  _iter.oops_do(cl);
+}
+
+void XStrongCLDsIterator::apply(CLDClosure* cl) {
+  XStatTimer timer(XSubPhaseConcurrentRootsClassLoaderDataGraph);
+  ClassLoaderDataGraph::always_strong_cld_do(cl);
+}
+
+XJavaThreadsIterator::XJavaThreadsIterator() :
+    _threads(),
+    _claimed(0) {}
+
+uint XJavaThreadsIterator::claim() {
+  return Atomic::fetch_and_add(&_claimed, 1u);
+}
+
+void XJavaThreadsIterator::apply(ThreadClosure* cl) {
+  XStatTimer timer(XSubPhaseConcurrentRootsJavaThreads);
+
+  // The resource mark is needed because interpreter oop maps are
+  // not reused in concurrent mode. Instead, they are temporary and
+  // resource allocated.
+  ResourceMark                 _rm;
+
+  for (uint i = claim(); i < _threads.length(); i = claim()) {
+    cl->do_thread(_threads.thread_at(i));
+  }
+}
+
+XNMethodsIterator::XNMethodsIterator() {
+  if (!ClassUnloading) {
+    XNMethod::nmethods_do_begin();
+  }
+}
+
+XNMethodsIterator::~XNMethodsIterator() {
+  if (!ClassUnloading) {
+    XNMethod::nmethods_do_end();
+  }
+}
+
+void XNMethodsIterator::apply(NMethodClosure* cl) {
+  XStatTimer timer(XSubPhaseConcurrentRootsCodeCache);
+  XNMethod::nmethods_do(cl);
+}
+
+XRootsIterator::XRootsIterator(int cld_claim) {
+  if (cld_claim != ClassLoaderData::_claim_none) {
+    ClassLoaderDataGraph::verify_claimed_marks_cleared(cld_claim);
+  }
+}
+
+void XRootsIterator::apply(OopClosure* cl,
+                           CLDClosure* cld_cl,
+                           ThreadClosure* thread_cl,
+                           NMethodClosure* nm_cl) {
+  _oop_storage_set.apply(cl);
+  _class_loader_data_graph.apply(cld_cl);
+  _java_threads.apply(thread_cl);
+  if (!ClassUnloading) {
+    _nmethods.apply(nm_cl);
+  }
+}
+
+XWeakOopStorageSetIterator::XWeakOopStorageSetIterator() :
+    _iter() {}
+
+void XWeakOopStorageSetIterator::apply(OopClosure* cl) {
+  XStatTimer timer(XSubPhaseConcurrentWeakRootsOopStorageSet);
+  _iter.oops_do(cl);
+}
+
+void XWeakOopStorageSetIterator::report_num_dead() {
+  _iter.report_num_dead();
+}
+
+void XWeakRootsIterator::report_num_dead() {
+  _oop_storage_set.iter().report_num_dead();
+}
+
+void XWeakRootsIterator::apply(OopClosure* cl) {
+  _oop_storage_set.apply(cl);
+}
diff --git a/src/hotspot/share/gc/x/xRootsIterator.hpp b/src/hotspot/share/gc/x/xRootsIterator.hpp
new file mode 100644
index 00000000000..9adc4c02938
--- /dev/null
+++ b/src/hotspot/share/gc/x/xRootsIterator.hpp
@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XROOTSITERATOR_HPP
+#define SHARE_GC_X_XROOTSITERATOR_HPP
+
+#include "gc/shared/oopStorageSetParState.hpp"
+#include "logging/log.hpp"
+#include "memory/iterator.hpp"
+#include "runtime/threadSMR.hpp"
+
+template <typename Iterator>
+class XParallelApply {
+private:
+  Iterator      _iter;
+  volatile bool _completed;
+
+public:
+  XParallelApply() :
+      _iter(),
+      _completed(false) {}
+
+  template <typename ClosureType>
+  void apply(ClosureType* cl);
+
+  Iterator& iter() {
+    return _iter;
+  }
+};
+
+class XStrongOopStorageSetIterator {
+  OopStorageSetStrongParState<true /* concurrent */, false /* is_const */> _iter;
+
+public:
+  XStrongOopStorageSetIterator();
+
+  void apply(OopClosure* cl);
+};
+
+class XStrongCLDsIterator {
+public:
+  void apply(CLDClosure* cl);
+};
+
+class XJavaThreadsIterator {
+private:
+  ThreadsListHandle _threads;
+  volatile uint     _claimed;
+
+  uint claim();
+
+public:
+  XJavaThreadsIterator();
+
+  void apply(ThreadClosure* cl);
+};
+
+class XNMethodsIterator {
+public:
+  XNMethodsIterator();
+  ~XNMethodsIterator();
+
+  void apply(NMethodClosure* cl);
+};
+
+class XRootsIterator {
+private:
+  XParallelApply<XStrongOopStorageSetIterator> _oop_storage_set;
+  XParallelApply<XStrongCLDsIterator>          _class_loader_data_graph;
+  XParallelApply<XJavaThreadsIterator>         _java_threads;
+  XParallelApply<XNMethodsIterator>            _nmethods;
+
+public:
+  XRootsIterator(int cld_claim);
+
+  void apply(OopClosure* cl,
+             CLDClosure* cld_cl,
+             ThreadClosure* thread_cl,
+             NMethodClosure* nm_cl);
+};
+
+class XWeakOopStorageSetIterator {
+private:
+  OopStorageSetWeakParState<true /* concurrent */, false /* is_const */> _iter;
+
+public:
+  XWeakOopStorageSetIterator();
+
+  void apply(OopClosure* cl);
+
+  void report_num_dead();
+};
+
+class XWeakRootsIterator {
+private:
+  XParallelApply<XWeakOopStorageSetIterator> _oop_storage_set;
+
+public:
+  void apply(OopClosure* cl);
+
+  void report_num_dead();
+};
+
+#endif // SHARE_GC_X_XROOTSITERATOR_HPP
diff --git a/src/hotspot/share/gc/x/xRuntimeWorkers.cpp b/src/hotspot/share/gc/x/xRuntimeWorkers.cpp
new file mode 100644
index 00000000000..d7e4a1262fc
--- /dev/null
+++ b/src/hotspot/share/gc/x/xRuntimeWorkers.cpp
@@ -0,0 +1,85 @@
+/*
+ * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/xLock.inline.hpp"
+#include "gc/x/xRuntimeWorkers.hpp"
+#include "gc/x/xTask.hpp"
+#include "gc/x/xThread.hpp"
+#include "runtime/java.hpp"
+
+class XRuntimeWorkersInitializeTask : public WorkerTask {
+private:
+  const uint     _nworkers;
+  uint           _started;
+  XConditionLock _lock;
+
+public:
+  XRuntimeWorkersInitializeTask(uint nworkers) :
+      WorkerTask("XRuntimeWorkersInitializeTask"),
+      _nworkers(nworkers),
+      _started(0),
+      _lock() {}
+
+  virtual void work(uint worker_id) {
+    // Wait for all threads to start
+    XLocker<XConditionLock> locker(&_lock);
+    if (++_started == _nworkers) {
+      // All threads started
+      _lock.notify_all();
+    } else {
+      while (_started != _nworkers) {
+        _lock.wait();
+      }
+    }
+  }
+};
+
+XRuntimeWorkers::XRuntimeWorkers() :
+    _workers("RuntimeWorker",
+             ParallelGCThreads) {
+
+  log_info_p(gc, init)("Runtime Workers: %u", _workers.max_workers());
+
+  // Initialize worker threads
+  _workers.initialize_workers();
+  _workers.set_active_workers(_workers.max_workers());
+  if (_workers.active_workers() != _workers.max_workers()) {
+    vm_exit_during_initialization("Failed to create XRuntimeWorkers");
+  }
+
+  // Execute task to reduce latency in early safepoints,
+  // which otherwise would have to take on any warmup costs.
+  XRuntimeWorkersInitializeTask task(_workers.max_workers());
+  _workers.run_task(&task);
+}
+
+WorkerThreads* XRuntimeWorkers::workers() {
+  return &_workers;
+}
+
+void XRuntimeWorkers::threads_do(ThreadClosure* tc) const {
+  _workers.threads_do(tc);
+}
diff --git a/src/hotspot/share/gc/x/xRuntimeWorkers.hpp b/src/hotspot/share/gc/x/xRuntimeWorkers.hpp
new file mode 100644
index 00000000000..114521d6506
--- /dev/null
+++ b/src/hotspot/share/gc/x/xRuntimeWorkers.hpp
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XRUNTIMEWORKERS_HPP
+#define SHARE_GC_X_XRUNTIMEWORKERS_HPP
+
+#include "gc/shared/workerThread.hpp"
+
+class ThreadClosure;
+
+class XRuntimeWorkers {
+private:
+  WorkerThreads _workers;
+
+public:
+  XRuntimeWorkers();
+
+  WorkerThreads* workers();
+
+  void threads_do(ThreadClosure* tc) const;
+};
+
+#endif // SHARE_GC_X_XRUNTIMEWORKERS_HPP
diff --git a/src/hotspot/share/gc/x/xSafeDelete.hpp b/src/hotspot/share/gc/x/xSafeDelete.hpp
new file mode 100644
index 00000000000..c41a38ce187
--- /dev/null
+++ b/src/hotspot/share/gc/x/xSafeDelete.hpp
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XSAFEDELETE_HPP
+#define SHARE_GC_X_XSAFEDELETE_HPP
+
+#include "gc/x/xArray.hpp"
+#include "gc/x/xLock.hpp"
+
+#include <type_traits>
+
+template <typename T>
+class XSafeDeleteImpl {
+private:
+  using ItemT = std::remove_extent_t<T>;
+
+  XLock*         _lock;
+  uint64_t       _enabled;
+  XArray<ItemT*> _deferred;
+
+  bool deferred_delete(ItemT* item);
+  void immediate_delete(ItemT* item);
+
+public:
+  XSafeDeleteImpl(XLock* lock);
+
+  void enable_deferred_delete();
+  void disable_deferred_delete();
+
+  void operator()(ItemT* item);
+};
+
+template <typename T>
+class XSafeDelete : public XSafeDeleteImpl<T> {
+private:
+  XLock _lock;
+
+public:
+  XSafeDelete();
+};
+
+template <typename T>
+class XSafeDeleteNoLock : public XSafeDeleteImpl<T> {
+public:
+  XSafeDeleteNoLock();
+};
+
+#endif // SHARE_GC_X_XSAFEDELETE_HPP
diff --git a/src/hotspot/share/gc/x/xSafeDelete.inline.hpp b/src/hotspot/share/gc/x/xSafeDelete.inline.hpp
new file mode 100644
index 00000000000..6c8417593d4
--- /dev/null
+++ b/src/hotspot/share/gc/x/xSafeDelete.inline.hpp
@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XSAFEDELETE_INLINE_HPP
+#define SHARE_GC_X_XSAFEDELETE_INLINE_HPP
+
+#include "gc/x/xSafeDelete.hpp"
+
+#include "gc/x/xArray.inline.hpp"
+#include "utilities/debug.hpp"
+
+#include <type_traits>
+
+template <typename T>
+XSafeDeleteImpl<T>::XSafeDeleteImpl(XLock* lock) :
+    _lock(lock),
+    _enabled(0),
+    _deferred() {}
+
+template <typename T>
+bool XSafeDeleteImpl<T>::deferred_delete(ItemT* item) {
+  XLocker<XLock> locker(_lock);
+  if (_enabled > 0) {
+    _deferred.append(item);
+    return true;
+  }
+
+  return false;
+}
+
+template <typename T>
+void XSafeDeleteImpl<T>::immediate_delete(ItemT* item) {
+  if (std::is_array<T>::value) {
+    delete [] item;
+  } else {
+    delete item;
+  }
+}
+
+template <typename T>
+void XSafeDeleteImpl<T>::enable_deferred_delete() {
+  XLocker<XLock> locker(_lock);
+  _enabled++;
+}
+
+template <typename T>
+void XSafeDeleteImpl<T>::disable_deferred_delete() {
+  XArray<ItemT*> deferred;
+
+  {
+    XLocker<XLock> locker(_lock);
+    assert(_enabled > 0, "Invalid state");
+    if (--_enabled == 0) {
+      deferred.swap(&_deferred);
+    }
+  }
+
+  XArrayIterator<ItemT*> iter(&deferred);
+  for (ItemT* item; iter.next(&item);) {
+    immediate_delete(item);
+  }
+}
+
+template <typename T>
+void XSafeDeleteImpl<T>::operator()(ItemT* item) {
+  if (!deferred_delete(item)) {
+    immediate_delete(item);
+  }
+}
+
+template <typename T>
+XSafeDelete<T>::XSafeDelete() :
+    XSafeDeleteImpl<T>(&_lock),
+    _lock() {}
+
+template <typename T>
+XSafeDeleteNoLock<T>::XSafeDeleteNoLock() :
+    XSafeDeleteImpl<T>(NULL) {}
+
+#endif // SHARE_GC_X_XSAFEDELETE_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xServiceability.cpp b/src/hotspot/share/gc/x/xServiceability.cpp
new file mode 100644
index 00000000000..6882896dfa9
--- /dev/null
+++ b/src/hotspot/share/gc/x/xServiceability.cpp
@@ -0,0 +1,177 @@
+/*
+ * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/generationCounters.hpp"
+#include "gc/shared/hSpaceCounters.hpp"
+#include "gc/x/xCollectedHeap.hpp"
+#include "gc/x/xHeap.inline.hpp"
+#include "gc/x/xServiceability.hpp"
+#include "memory/metaspaceCounters.hpp"
+#include "runtime/perfData.hpp"
+
+class XGenerationCounters : public GenerationCounters {
+public:
+  XGenerationCounters(const char* name, int ordinal, int spaces,
+                      size_t min_capacity, size_t max_capacity, size_t curr_capacity) :
+      GenerationCounters(name, ordinal, spaces,
+                         min_capacity, max_capacity, curr_capacity) {}
+
+  void update_capacity(size_t capacity) {
+    _current_size->set_value(capacity);
+  }
+};
+
+// Class to expose perf counters used by jstat.
+class XServiceabilityCounters : public CHeapObj<mtGC> {
+private:
+  XGenerationCounters _generation_counters;
+  HSpaceCounters      _space_counters;
+  CollectorCounters   _collector_counters;
+
+public:
+  XServiceabilityCounters(size_t min_capacity, size_t max_capacity);
+
+  CollectorCounters* collector_counters();
+
+  void update_sizes();
+};
+
+XServiceabilityCounters::XServiceabilityCounters(size_t min_capacity, size_t max_capacity) :
+    // generation.1
+    _generation_counters("old"        /* name */,
+                         1            /* ordinal */,
+                         1            /* spaces */,
+                         min_capacity /* min_capacity */,
+                         max_capacity /* max_capacity */,
+                         min_capacity /* curr_capacity */),
+    // generation.1.space.0
+    _space_counters(_generation_counters.name_space(),
+                    "space"      /* name */,
+                    0            /* ordinal */,
+                    max_capacity /* max_capacity */,
+                    min_capacity /* init_capacity */),
+    // gc.collector.2
+    _collector_counters("Z concurrent cycle pauses" /* name */,
+                        2                           /* ordinal */) {}
+
+CollectorCounters* XServiceabilityCounters::collector_counters() {
+  return &_collector_counters;
+}
+
+void XServiceabilityCounters::update_sizes() {
+  if (UsePerfData) {
+    const size_t capacity = XHeap::heap()->capacity();
+    const size_t used = MIN2(XHeap::heap()->used(), capacity);
+
+    _generation_counters.update_capacity(capacity);
+    _space_counters.update_capacity(capacity);
+    _space_counters.update_used(used);
+
+    MetaspaceCounters::update_performance_counters();
+  }
+}
+
+XServiceabilityMemoryPool::XServiceabilityMemoryPool(size_t min_capacity, size_t max_capacity) :
+    CollectedMemoryPool("ZHeap",
+                        min_capacity,
+                        max_capacity,
+                        true /* support_usage_threshold */) {}
+
+size_t XServiceabilityMemoryPool::used_in_bytes() {
+  return XHeap::heap()->used();
+}
+
+MemoryUsage XServiceabilityMemoryPool::get_memory_usage() {
+  const size_t committed = XHeap::heap()->capacity();
+  const size_t used      = MIN2(XHeap::heap()->used(), committed);
+
+  return MemoryUsage(initial_size(), used, committed, max_size());
+}
+
+XServiceabilityMemoryManager::XServiceabilityMemoryManager(const char* name,
+                                                           XServiceabilityMemoryPool* pool) :
+    GCMemoryManager(name) {
+  add_pool(pool);
+}
+
+XServiceability::XServiceability(size_t min_capacity, size_t max_capacity) :
+    _min_capacity(min_capacity),
+    _max_capacity(max_capacity),
+    _memory_pool(_min_capacity, _max_capacity),
+    _cycle_memory_manager("ZGC Cycles", &_memory_pool),
+    _pause_memory_manager("ZGC Pauses", &_memory_pool),
+    _counters(NULL) {}
+
+void XServiceability::initialize() {
+  _counters = new XServiceabilityCounters(_min_capacity, _max_capacity);
+}
+
+MemoryPool* XServiceability::memory_pool() {
+  return &_memory_pool;
+}
+
+GCMemoryManager* XServiceability::cycle_memory_manager() {
+  return &_cycle_memory_manager;
+}
+
+GCMemoryManager* XServiceability::pause_memory_manager() {
+  return &_pause_memory_manager;
+}
+
+XServiceabilityCounters* XServiceability::counters() {
+  return _counters;
+}
+
+XServiceabilityCycleTracer::XServiceabilityCycleTracer() :
+    _memory_manager_stats(XHeap::heap()->serviceability_cycle_memory_manager(),
+                          XCollectedHeap::heap()->gc_cause(),
+                          "end of GC cycle",
+                          true  /* allMemoryPoolsAffected */,
+                          true  /* recordGCBeginTime */,
+                          true  /* recordPreGCUsage */,
+                          true  /* recordPeakUsage */,
+                          true  /* recordPostGCUsage */,
+                          true  /* recordAccumulatedGCTime */,
+                          true  /* recordGCEndTime */,
+                          true  /* countCollection */) {}
+
+XServiceabilityPauseTracer::XServiceabilityPauseTracer() :
+    _svc_gc_marker(SvcGCMarker::CONCURRENT),
+    _counters_stats(XHeap::heap()->serviceability_counters()->collector_counters()),
+    _memory_manager_stats(XHeap::heap()->serviceability_pause_memory_manager(),
+                          XCollectedHeap::heap()->gc_cause(),
+                          "end of GC pause",
+                          true  /* allMemoryPoolsAffected */,
+                          true  /* recordGCBeginTime */,
+                          false /* recordPreGCUsage */,
+                          false /* recordPeakUsage */,
+                          false /* recordPostGCUsage */,
+                          true  /* recordAccumulatedGCTime */,
+                          true  /* recordGCEndTime */,
+                          true  /* countCollection */) {}
+
+XServiceabilityPauseTracer::~XServiceabilityPauseTracer()  {
+  XHeap::heap()->serviceability_counters()->update_sizes();
+  MemoryService::track_memory_usage();
+}
diff --git a/src/hotspot/share/gc/x/xServiceability.hpp b/src/hotspot/share/gc/x/xServiceability.hpp
new file mode 100644
index 00000000000..d8e2fc9ba79
--- /dev/null
+++ b/src/hotspot/share/gc/x/xServiceability.hpp
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XSERVICEABILITY_HPP
+#define SHARE_GC_X_XSERVICEABILITY_HPP
+
+#include "gc/shared/collectorCounters.hpp"
+#include "gc/shared/gcVMOperations.hpp"
+#include "memory/allocation.hpp"
+#include "services/memoryManager.hpp"
+#include "services/memoryPool.hpp"
+#include "services/memoryService.hpp"
+
+class XServiceabilityCounters;
+
+class XServiceabilityMemoryPool : public CollectedMemoryPool {
+public:
+  XServiceabilityMemoryPool(size_t min_capacity, size_t max_capacity);
+
+  virtual size_t used_in_bytes();
+  virtual MemoryUsage get_memory_usage();
+};
+
+class XServiceabilityMemoryManager : public GCMemoryManager {
+public:
+  XServiceabilityMemoryManager(const char* name,
+                               XServiceabilityMemoryPool* pool);
+};
+
+class XServiceability {
+private:
+  const size_t                 _min_capacity;
+  const size_t                 _max_capacity;
+  XServiceabilityMemoryPool    _memory_pool;
+  XServiceabilityMemoryManager _cycle_memory_manager;
+  XServiceabilityMemoryManager _pause_memory_manager;
+  XServiceabilityCounters*     _counters;
+
+public:
+  XServiceability(size_t min_capacity, size_t max_capacity);
+
+  void initialize();
+
+  MemoryPool* memory_pool();
+  GCMemoryManager* cycle_memory_manager();
+  GCMemoryManager* pause_memory_manager();
+  XServiceabilityCounters* counters();
+};
+
+class XServiceabilityCycleTracer : public StackObj {
+private:
+  TraceMemoryManagerStats _memory_manager_stats;
+
+public:
+  XServiceabilityCycleTracer();
+};
+
+class XServiceabilityPauseTracer : public StackObj {
+private:
+  SvcGCMarker             _svc_gc_marker;
+  TraceCollectorStats     _counters_stats;
+  TraceMemoryManagerStats _memory_manager_stats;
+
+public:
+  XServiceabilityPauseTracer();
+  ~XServiceabilityPauseTracer();
+};
+
+#endif // SHARE_GC_X_XSERVICEABILITY_HPP
diff --git a/src/hotspot/share/gc/x/xStackWatermark.cpp b/src/hotspot/share/gc/x/xStackWatermark.cpp
new file mode 100644
index 00000000000..7be799f74f0
--- /dev/null
+++ b/src/hotspot/share/gc/x/xStackWatermark.cpp
@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xAddress.hpp"
+#include "gc/x/xBarrier.inline.hpp"
+#include "gc/x/xStackWatermark.hpp"
+#include "gc/x/xThread.inline.hpp"
+#include "gc/x/xThreadLocalAllocBuffer.hpp"
+#include "gc/x/xThreadLocalData.hpp"
+#include "gc/x/xVerify.hpp"
+#include "memory/resourceArea.inline.hpp"
+#include "runtime/frame.inline.hpp"
+#include "utilities/preserveException.hpp"
+
+XOnStackCodeBlobClosure::XOnStackCodeBlobClosure() :
+    _bs_nm(BarrierSet::barrier_set()->barrier_set_nmethod()) {}
+
+void XOnStackCodeBlobClosure::do_code_blob(CodeBlob* cb) {
+  nmethod* const nm = cb->as_nmethod_or_null();
+  if (nm != NULL) {
+    const bool result = _bs_nm->nmethod_entry_barrier(nm);
+    assert(result, "NMethod on-stack must be alive");
+  }
+}
+
+ThreadLocalAllocStats& XStackWatermark::stats() {
+  return _stats;
+}
+
+uint32_t XStackWatermark::epoch_id() const {
+  return *XAddressBadMaskHighOrderBitsAddr;
+}
+
+XStackWatermark::XStackWatermark(JavaThread* jt) :
+    StackWatermark(jt, StackWatermarkKind::gc, *XAddressBadMaskHighOrderBitsAddr),
+    _jt_cl(),
+    _cb_cl(),
+    _stats() {}
+
+OopClosure* XStackWatermark::closure_from_context(void* context) {
+  if (context != NULL) {
+    assert(XThread::is_worker(), "Unexpected thread passing in context: " PTR_FORMAT, p2i(context));
+    return reinterpret_cast<OopClosure*>(context);
+  } else {
+    return &_jt_cl;
+  }
+}
+
+void XStackWatermark::start_processing_impl(void* context) {
+  // Verify the head (no_frames) of the thread is bad before fixing it.
+  XVerify::verify_thread_head_bad(_jt);
+
+  // Process the non-frame part of the thread
+  _jt->oops_do_no_frames(closure_from_context(context), &_cb_cl);
+  XThreadLocalData::do_invisible_root(_jt, XBarrier::load_barrier_on_invisible_root_oop_field);
+
+  // Verification of frames is done after processing of the "head" (no_frames).
+  // The reason is that the exception oop is fiddled with during frame processing.
+  XVerify::verify_thread_frames_bad(_jt);
+
+  // Update thread local address bad mask
+  XThreadLocalData::set_address_bad_mask(_jt, XAddressBadMask);
+
+  // Retire TLAB
+  if (XGlobalPhase == XPhaseMark) {
+    XThreadLocalAllocBuffer::retire(_jt, &_stats);
+  } else {
+    XThreadLocalAllocBuffer::remap(_jt);
+  }
+
+  // Publishes the processing start to concurrent threads
+  StackWatermark::start_processing_impl(context);
+}
+
+void XStackWatermark::process(const frame& fr, RegisterMap& register_map, void* context) {
+  XVerify::verify_frame_bad(fr, register_map);
+  fr.oops_do(closure_from_context(context), &_cb_cl, &register_map, DerivedPointerIterationMode::_directly);
+}
diff --git a/src/hotspot/share/gc/x/xStackWatermark.hpp b/src/hotspot/share/gc/x/xStackWatermark.hpp
new file mode 100644
index 00000000000..b9f39d0d1d8
--- /dev/null
+++ b/src/hotspot/share/gc/x/xStackWatermark.hpp
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XSTACKWATERMARK_HPP
+#define SHARE_GC_X_XSTACKWATERMARK_HPP
+
+#include "gc/shared/barrierSet.hpp"
+#include "gc/shared/barrierSetNMethod.hpp"
+#include "gc/shared/threadLocalAllocBuffer.hpp"
+#include "gc/x/xBarrier.hpp"
+#include "memory/allocation.hpp"
+#include "memory/iterator.hpp"
+#include "oops/oopsHierarchy.hpp"
+#include "runtime/stackWatermark.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class frame;
+class JavaThread;
+
+class XOnStackCodeBlobClosure : public CodeBlobClosure {
+private:
+  BarrierSetNMethod* _bs_nm;
+
+  virtual void do_code_blob(CodeBlob* cb);
+
+public:
+  XOnStackCodeBlobClosure();
+};
+
+class XStackWatermark : public StackWatermark {
+private:
+  XLoadBarrierOopClosure  _jt_cl;
+  XOnStackCodeBlobClosure _cb_cl;
+  ThreadLocalAllocStats   _stats;
+
+  OopClosure* closure_from_context(void* context);
+
+  virtual uint32_t epoch_id() const;
+  virtual void start_processing_impl(void* context);
+  virtual void process(const frame& fr, RegisterMap& register_map, void* context);
+
+public:
+  XStackWatermark(JavaThread* jt);
+
+  ThreadLocalAllocStats& stats();
+};
+
+#endif // SHARE_GC_X_XSTACKWATERMARK_HPP
diff --git a/src/hotspot/share/gc/x/xStat.cpp b/src/hotspot/share/gc/x/xStat.cpp
new file mode 100644
index 00000000000..beb90cc1740
--- /dev/null
+++ b/src/hotspot/share/gc/x/xStat.cpp
@@ -0,0 +1,1513 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/xAbort.inline.hpp"
+#include "gc/x/xCollectedHeap.hpp"
+#include "gc/x/xCPU.inline.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xNMethodTable.hpp"
+#include "gc/x/xPageAllocator.inline.hpp"
+#include "gc/x/xRelocationSetSelector.inline.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xThread.inline.hpp"
+#include "gc/x/xTracer.inline.hpp"
+#include "gc/x/xUtils.hpp"
+#include "memory/metaspaceUtils.hpp"
+#include "memory/resourceArea.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/os.hpp"
+#include "runtime/timer.hpp"
+#include "utilities/align.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/ticks.hpp"
+
+#define XSIZE_FMT                       SIZE_FORMAT "M(%.0f%%)"
+#define XSIZE_ARGS_WITH_MAX(size, max)  ((size) / M), (percent_of(size, max))
+#define XSIZE_ARGS(size)                XSIZE_ARGS_WITH_MAX(size, XStatHeap::max_capacity())
+
+#define XTABLE_ARGS_NA                  "%9s", "-"
+#define XTABLE_ARGS(size)               SIZE_FORMAT_W(8) "M (%.0f%%)", \
+                                        ((size) / M), (percent_of(size, XStatHeap::max_capacity()))
+
+//
+// Stat sampler/counter data
+//
+struct XStatSamplerData {
+  uint64_t _nsamples;
+  uint64_t _sum;
+  uint64_t _max;
+
+  XStatSamplerData() :
+    _nsamples(0),
+    _sum(0),
+    _max(0) {}
+
+  void add(const XStatSamplerData& new_sample) {
+    _nsamples += new_sample._nsamples;
+    _sum += new_sample._sum;
+    _max = MAX2(_max, new_sample._max);
+  }
+};
+
+struct XStatCounterData {
+  uint64_t _counter;
+
+  XStatCounterData() :
+    _counter(0) {}
+};
+
+//
+// Stat sampler history
+//
+template <size_t size>
+class XStatSamplerHistoryInterval {
+private:
+  size_t           _next;
+  XStatSamplerData _samples[size];
+  XStatSamplerData _accumulated;
+  XStatSamplerData _total;
+
+public:
+  XStatSamplerHistoryInterval() :
+      _next(0),
+      _samples(),
+      _accumulated(),
+      _total() {}
+
+  bool add(const XStatSamplerData& new_sample) {
+    // Insert sample
+    const XStatSamplerData old_sample = _samples[_next];
+    _samples[_next] = new_sample;
+
+    // Adjust accumulated
+    _accumulated._nsamples += new_sample._nsamples;
+    _accumulated._sum += new_sample._sum;
+    _accumulated._max = MAX2(_accumulated._max, new_sample._max);
+
+    // Adjust total
+    _total._nsamples -= old_sample._nsamples;
+    _total._sum -= old_sample._sum;
+    _total._nsamples += new_sample._nsamples;
+    _total._sum += new_sample._sum;
+    if (_total._max < new_sample._max) {
+      // Found new max
+      _total._max = new_sample._max;
+    } else if (_total._max == old_sample._max) {
+      // Removed old max, reset and find new max
+      _total._max = 0;
+      for (size_t i = 0; i < size; i++) {
+        if (_total._max < _samples[i]._max) {
+          _total._max = _samples[i]._max;
+        }
+      }
+    }
+
+    // Adjust next
+    if (++_next == size) {
+      _next = 0;
+
+      // Clear accumulated
+      const XStatSamplerData zero;
+      _accumulated = zero;
+
+      // Became full
+      return true;
+    }
+
+    // Not yet full
+    return false;
+  }
+
+  const XStatSamplerData& total() const {
+    return _total;
+  }
+
+  const XStatSamplerData& accumulated() const {
+    return _accumulated;
+  }
+};
+
+class XStatSamplerHistory : public CHeapObj<mtGC> {
+private:
+  XStatSamplerHistoryInterval<10> _10seconds;
+  XStatSamplerHistoryInterval<60> _10minutes;
+  XStatSamplerHistoryInterval<60> _10hours;
+  XStatSamplerData                _total;
+
+  uint64_t avg(uint64_t sum, uint64_t nsamples) const {
+    return (nsamples > 0) ? sum / nsamples : 0;
+  }
+
+public:
+  XStatSamplerHistory() :
+      _10seconds(),
+      _10minutes(),
+      _10hours(),
+      _total() {}
+
+  void add(const XStatSamplerData& new_sample) {
+    if (_10seconds.add(new_sample)) {
+      if (_10minutes.add(_10seconds.total())) {
+        if (_10hours.add(_10minutes.total())) {
+          _total.add(_10hours.total());
+        }
+      }
+    }
+  }
+
+  uint64_t avg_10_seconds() const {
+    const uint64_t sum      = _10seconds.total()._sum;
+    const uint64_t nsamples = _10seconds.total()._nsamples;
+    return avg(sum, nsamples);
+  }
+
+  uint64_t avg_10_minutes() const {
+    const uint64_t sum      = _10seconds.accumulated()._sum +
+                              _10minutes.total()._sum;
+    const uint64_t nsamples = _10seconds.accumulated()._nsamples +
+                              _10minutes.total()._nsamples;
+    return avg(sum, nsamples);
+  }
+
+  uint64_t avg_10_hours() const {
+    const uint64_t sum      = _10seconds.accumulated()._sum +
+                              _10minutes.accumulated()._sum +
+                              _10hours.total()._sum;
+    const uint64_t nsamples = _10seconds.accumulated()._nsamples +
+                              _10minutes.accumulated()._nsamples +
+                              _10hours.total()._nsamples;
+    return avg(sum, nsamples);
+  }
+
+  uint64_t avg_total() const {
+    const uint64_t sum      = _10seconds.accumulated()._sum +
+                              _10minutes.accumulated()._sum +
+                              _10hours.accumulated()._sum +
+                              _total._sum;
+    const uint64_t nsamples = _10seconds.accumulated()._nsamples +
+                              _10minutes.accumulated()._nsamples +
+                              _10hours.accumulated()._nsamples +
+                              _total._nsamples;
+    return avg(sum, nsamples);
+  }
+
+  uint64_t max_10_seconds() const {
+    return _10seconds.total()._max;
+  }
+
+  uint64_t max_10_minutes() const {
+    return MAX2(_10seconds.accumulated()._max,
+                _10minutes.total()._max);
+  }
+
+  uint64_t max_10_hours() const {
+    return MAX3(_10seconds.accumulated()._max,
+                _10minutes.accumulated()._max,
+                _10hours.total()._max);
+  }
+
+  uint64_t max_total() const {
+    return MAX4(_10seconds.accumulated()._max,
+                _10minutes.accumulated()._max,
+                _10hours.accumulated()._max,
+                _total._max);
+  }
+};
+
+//
+// Stat unit printers
+//
+void XStatUnitTime(LogTargetHandle log, const XStatSampler& sampler, const XStatSamplerHistory& history) {
+  log.print(" %10s: %-41s "
+            "%9.3f / %-9.3f "
+            "%9.3f / %-9.3f "
+            "%9.3f / %-9.3f "
+            "%9.3f / %-9.3f   ms",
+            sampler.group(),
+            sampler.name(),
+            TimeHelper::counter_to_millis(history.avg_10_seconds()),
+            TimeHelper::counter_to_millis(history.max_10_seconds()),
+            TimeHelper::counter_to_millis(history.avg_10_minutes()),
+            TimeHelper::counter_to_millis(history.max_10_minutes()),
+            TimeHelper::counter_to_millis(history.avg_10_hours()),
+            TimeHelper::counter_to_millis(history.max_10_hours()),
+            TimeHelper::counter_to_millis(history.avg_total()),
+            TimeHelper::counter_to_millis(history.max_total()));
+}
+
+void XStatUnitBytes(LogTargetHandle log, const XStatSampler& sampler, const XStatSamplerHistory& history) {
+  log.print(" %10s: %-41s "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) "   MB",
+            sampler.group(),
+            sampler.name(),
+            history.avg_10_seconds() / M,
+            history.max_10_seconds() / M,
+            history.avg_10_minutes() / M,
+            history.max_10_minutes() / M,
+            history.avg_10_hours() / M,
+            history.max_10_hours() / M,
+            history.avg_total() / M,
+            history.max_total() / M);
+}
+
+void XStatUnitThreads(LogTargetHandle log, const XStatSampler& sampler, const XStatSamplerHistory& history) {
+  log.print(" %10s: %-41s "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) "   threads",
+            sampler.group(),
+            sampler.name(),
+            history.avg_10_seconds(),
+            history.max_10_seconds(),
+            history.avg_10_minutes(),
+            history.max_10_minutes(),
+            history.avg_10_hours(),
+            history.max_10_hours(),
+            history.avg_total(),
+            history.max_total());
+}
+
+void XStatUnitBytesPerSecond(LogTargetHandle log, const XStatSampler& sampler, const XStatSamplerHistory& history) {
+  log.print(" %10s: %-41s "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) "   MB/s",
+            sampler.group(),
+            sampler.name(),
+            history.avg_10_seconds() / M,
+            history.max_10_seconds() / M,
+            history.avg_10_minutes() / M,
+            history.max_10_minutes() / M,
+            history.avg_10_hours() / M,
+            history.max_10_hours() / M,
+            history.avg_total() / M,
+            history.max_total() / M);
+}
+
+void XStatUnitOpsPerSecond(LogTargetHandle log, const XStatSampler& sampler, const XStatSamplerHistory& history) {
+  log.print(" %10s: %-41s "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
+            UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) "   ops/s",
+            sampler.group(),
+            sampler.name(),
+            history.avg_10_seconds(),
+            history.max_10_seconds(),
+            history.avg_10_minutes(),
+            history.max_10_minutes(),
+            history.avg_10_hours(),
+            history.max_10_hours(),
+            history.avg_total(),
+            history.max_total());
+}
+
+//
+// Stat value
+//
+uintptr_t XStatValue::_base = 0;
+uint32_t  XStatValue::_cpu_offset = 0;
+
+XStatValue::XStatValue(const char* group,
+                          const char* name,
+                          uint32_t id,
+                          uint32_t size) :
+    _group(group),
+    _name(name),
+    _id(id),
+    _offset(_cpu_offset) {
+  assert(_base == 0, "Already initialized");
+  _cpu_offset += size;
+}
+
+template <typename T>
+T* XStatValue::get_cpu_local(uint32_t cpu) const {
+  assert(_base != 0, "Not initialized");
+  const uintptr_t cpu_base = _base + (_cpu_offset * cpu);
+  const uintptr_t value_addr = cpu_base + _offset;
+  return (T*)value_addr;
+}
+
+void XStatValue::initialize() {
+  // Finalize and align CPU offset
+  _cpu_offset = align_up(_cpu_offset, (uint32_t)XCacheLineSize);
+
+  // Allocation aligned memory
+  const size_t size = _cpu_offset * XCPU::count();
+  _base = XUtils::alloc_aligned(XCacheLineSize, size);
+}
+
+const char* XStatValue::group() const {
+  return _group;
+}
+
+const char* XStatValue::name() const {
+  return _name;
+}
+
+uint32_t XStatValue::id() const {
+  return _id;
+}
+
+//
+// Stat iterable value
+//
+
+template <typename T>
+XStatIterableValue<T>::XStatIterableValue(const char* group,
+                                          const char* name,
+                                          uint32_t size) :
+    XStatValue(group, name, _count++, size),
+    _next(insert()) {}
+
+template <typename T>
+T* XStatIterableValue<T>::insert() const {
+  T* const next = _first;
+  _first = (T*)this;
+  return next;
+}
+
+template <typename T>
+void XStatIterableValue<T>::sort() {
+  T* first_unsorted = _first;
+  _first = NULL;
+
+  while (first_unsorted != NULL) {
+    T* const value = first_unsorted;
+    first_unsorted = value->_next;
+    value->_next = NULL;
+
+    T** current = &_first;
+
+    while (*current != NULL) {
+      // First sort by group, then by name
+      const int group_cmp = strcmp((*current)->group(), value->group());
+      if ((group_cmp > 0) || (group_cmp == 0 && strcmp((*current)->name(), value->name()) > 0)) {
+        break;
+      }
+
+      current = &(*current)->_next;
+    }
+    value->_next = *current;
+    *current = value;
+  }
+}
+
+//
+// Stat sampler
+//
+XStatSampler::XStatSampler(const char* group, const char* name, XStatUnitPrinter printer) :
+    XStatIterableValue<XStatSampler>(group, name, sizeof(XStatSamplerData)),
+    _printer(printer) {}
+
+XStatSamplerData* XStatSampler::get() const {
+  return get_cpu_local<XStatSamplerData>(XCPU::id());
+}
+
+XStatSamplerData XStatSampler::collect_and_reset() const {
+  XStatSamplerData all;
+
+  const uint32_t ncpus = XCPU::count();
+  for (uint32_t i = 0; i < ncpus; i++) {
+    XStatSamplerData* const cpu_data = get_cpu_local<XStatSamplerData>(i);
+    if (cpu_data->_nsamples > 0) {
+      const uint64_t nsamples = Atomic::xchg(&cpu_data->_nsamples, (uint64_t)0);
+      const uint64_t sum = Atomic::xchg(&cpu_data->_sum, (uint64_t)0);
+      const uint64_t max = Atomic::xchg(&cpu_data->_max, (uint64_t)0);
+      all._nsamples += nsamples;
+      all._sum += sum;
+      if (all._max < max) {
+        all._max = max;
+      }
+    }
+  }
+
+  return all;
+}
+
+XStatUnitPrinter XStatSampler::printer() const {
+  return _printer;
+}
+
+//
+// Stat counter
+//
+XStatCounter::XStatCounter(const char* group, const char* name, XStatUnitPrinter printer) :
+    XStatIterableValue<XStatCounter>(group, name, sizeof(XStatCounterData)),
+    _sampler(group, name, printer) {}
+
+XStatCounterData* XStatCounter::get() const {
+  return get_cpu_local<XStatCounterData>(XCPU::id());
+}
+
+void XStatCounter::sample_and_reset() const {
+  uint64_t counter = 0;
+
+  const uint32_t ncpus = XCPU::count();
+  for (uint32_t i = 0; i < ncpus; i++) {
+    XStatCounterData* const cpu_data = get_cpu_local<XStatCounterData>(i);
+    counter += Atomic::xchg(&cpu_data->_counter, (uint64_t)0);
+  }
+
+  XStatSample(_sampler, counter);
+}
+
+//
+// Stat unsampled counter
+//
+XStatUnsampledCounter::XStatUnsampledCounter(const char* name) :
+    XStatIterableValue<XStatUnsampledCounter>("Unsampled", name, sizeof(XStatCounterData)) {}
+
+XStatCounterData* XStatUnsampledCounter::get() const {
+  return get_cpu_local<XStatCounterData>(XCPU::id());
+}
+
+XStatCounterData XStatUnsampledCounter::collect_and_reset() const {
+  XStatCounterData all;
+
+  const uint32_t ncpus = XCPU::count();
+  for (uint32_t i = 0; i < ncpus; i++) {
+    XStatCounterData* const cpu_data = get_cpu_local<XStatCounterData>(i);
+    all._counter += Atomic::xchg(&cpu_data->_counter, (uint64_t)0);
+  }
+
+  return all;
+}
+
+//
+// Stat MMU (Minimum Mutator Utilization)
+//
+XStatMMUPause::XStatMMUPause() :
+    _start(0.0),
+    _end(0.0) {}
+
+XStatMMUPause::XStatMMUPause(const Ticks& start, const Ticks& end) :
+    _start(TimeHelper::counter_to_millis(start.value())),
+    _end(TimeHelper::counter_to_millis(end.value())) {}
+
+double XStatMMUPause::end() const {
+  return _end;
+}
+
+double XStatMMUPause::overlap(double start, double end) const {
+  const double start_max = MAX2(start, _start);
+  const double end_min = MIN2(end, _end);
+
+  if (end_min > start_max) {
+    // Overlap found
+    return end_min - start_max;
+  }
+
+  // No overlap
+  return 0.0;
+}
+
+size_t XStatMMU::_next = 0;
+size_t XStatMMU::_npauses = 0;
+XStatMMUPause XStatMMU::_pauses[200];
+double XStatMMU::_mmu_2ms = 100.0;
+double XStatMMU::_mmu_5ms = 100.0;
+double XStatMMU::_mmu_10ms = 100.0;
+double XStatMMU::_mmu_20ms = 100.0;
+double XStatMMU::_mmu_50ms = 100.0;
+double XStatMMU::_mmu_100ms = 100.0;
+
+const XStatMMUPause& XStatMMU::pause(size_t index) {
+  return _pauses[(_next - index - 1) % ARRAY_SIZE(_pauses)];
+}
+
+double XStatMMU::calculate_mmu(double time_slice) {
+  const double end = pause(0).end();
+  const double start = end - time_slice;
+  double time_paused = 0.0;
+
+  // Find all overlapping pauses
+  for (size_t i = 0; i < _npauses; i++) {
+    const double overlap = pause(i).overlap(start, end);
+    if (overlap == 0.0) {
+      // No overlap
+      break;
+    }
+
+    time_paused += overlap;
+  }
+
+  // Calculate MMU
+  const double time_mutator = time_slice - time_paused;
+  return percent_of(time_mutator, time_slice);
+}
+
+void XStatMMU::register_pause(const Ticks& start, const Ticks& end) {
+  // Add pause
+  const size_t index = _next++ % ARRAY_SIZE(_pauses);
+  _pauses[index] = XStatMMUPause(start, end);
+  _npauses = MIN2(_npauses + 1, ARRAY_SIZE(_pauses));
+
+  // Recalculate MMUs
+  _mmu_2ms    = MIN2(_mmu_2ms,   calculate_mmu(2));
+  _mmu_5ms    = MIN2(_mmu_5ms,   calculate_mmu(5));
+  _mmu_10ms   = MIN2(_mmu_10ms,  calculate_mmu(10));
+  _mmu_20ms   = MIN2(_mmu_20ms,  calculate_mmu(20));
+  _mmu_50ms   = MIN2(_mmu_50ms,  calculate_mmu(50));
+  _mmu_100ms  = MIN2(_mmu_100ms, calculate_mmu(100));
+}
+
+void XStatMMU::print() {
+  log_info(gc, mmu)("MMU: 2ms/%.1f%%, 5ms/%.1f%%, 10ms/%.1f%%, 20ms/%.1f%%, 50ms/%.1f%%, 100ms/%.1f%%",
+                    _mmu_2ms, _mmu_5ms, _mmu_10ms, _mmu_20ms, _mmu_50ms, _mmu_100ms);
+}
+
+//
+// Stat phases
+//
+ConcurrentGCTimer XStatPhase::_timer;
+
+XStatPhase::XStatPhase(const char* group, const char* name) :
+    _sampler(group, name, XStatUnitTime) {}
+
+void XStatPhase::log_start(LogTargetHandle log, bool thread) const {
+  if (!log.is_enabled()) {
+    return;
+  }
+
+  if (thread) {
+    ResourceMark rm;
+    log.print("%s (%s)", name(), Thread::current()->name());
+  } else {
+    log.print("%s", name());
+  }
+}
+
+void XStatPhase::log_end(LogTargetHandle log, const Tickspan& duration, bool thread) const {
+  if (!log.is_enabled()) {
+    return;
+  }
+
+  if (thread) {
+    ResourceMark rm;
+    log.print("%s (%s) %.3fms", name(), Thread::current()->name(), TimeHelper::counter_to_millis(duration.value()));
+  } else {
+    log.print("%s %.3fms", name(), TimeHelper::counter_to_millis(duration.value()));
+  }
+}
+
+ConcurrentGCTimer* XStatPhase::timer() {
+  return &_timer;
+}
+
+const char* XStatPhase::name() const {
+  return _sampler.name();
+}
+
+XStatPhaseCycle::XStatPhaseCycle(const char* name) :
+    XStatPhase("Collector", name) {}
+
+void XStatPhaseCycle::register_start(const Ticks& start) const {
+  timer()->register_gc_start(start);
+
+  XTracer::tracer()->report_gc_start(XCollectedHeap::heap()->gc_cause(), start);
+
+  XCollectedHeap::heap()->print_heap_before_gc();
+  XCollectedHeap::heap()->trace_heap_before_gc(XTracer::tracer());
+
+  log_info(gc, start)("Garbage Collection (%s)",
+                       GCCause::to_string(XCollectedHeap::heap()->gc_cause()));
+}
+
+void XStatPhaseCycle::register_end(const Ticks& start, const Ticks& end) const {
+  if (XAbort::should_abort()) {
+    log_info(gc)("Garbage Collection (%s) Aborted",
+                 GCCause::to_string(XCollectedHeap::heap()->gc_cause()));
+    return;
+  }
+
+  timer()->register_gc_end(end);
+
+  XCollectedHeap::heap()->print_heap_after_gc();
+  XCollectedHeap::heap()->trace_heap_after_gc(XTracer::tracer());
+
+  XTracer::tracer()->report_gc_end(end, timer()->time_partitions());
+
+  const Tickspan duration = end - start;
+  XStatSample(_sampler, duration.value());
+
+  XStatLoad::print();
+  XStatMMU::print();
+  XStatMark::print();
+  XStatNMethods::print();
+  XStatMetaspace::print();
+  XStatReferences::print();
+  XStatRelocation::print();
+  XStatHeap::print();
+
+  log_info(gc)("Garbage Collection (%s) " XSIZE_FMT "->" XSIZE_FMT,
+               GCCause::to_string(XCollectedHeap::heap()->gc_cause()),
+               XSIZE_ARGS(XStatHeap::used_at_mark_start()),
+               XSIZE_ARGS(XStatHeap::used_at_relocate_end()));
+}
+
+Tickspan XStatPhasePause::_max;
+
+XStatPhasePause::XStatPhasePause(const char* name) :
+    XStatPhase("Phase", name) {}
+
+const Tickspan& XStatPhasePause::max() {
+  return _max;
+}
+
+void XStatPhasePause::register_start(const Ticks& start) const {
+  timer()->register_gc_pause_start(name(), start);
+
+  LogTarget(Debug, gc, phases, start) log;
+  log_start(log);
+}
+
+void XStatPhasePause::register_end(const Ticks& start, const Ticks& end) const {
+  timer()->register_gc_pause_end(end);
+
+  const Tickspan duration = end - start;
+  XStatSample(_sampler, duration.value());
+
+  // Track max pause time
+  if (_max < duration) {
+    _max = duration;
+  }
+
+  // Track minimum mutator utilization
+  XStatMMU::register_pause(start, end);
+
+  LogTarget(Info, gc, phases) log;
+  log_end(log, duration);
+}
+
+XStatPhaseConcurrent::XStatPhaseConcurrent(const char* name) :
+    XStatPhase("Phase", name) {}
+
+void XStatPhaseConcurrent::register_start(const Ticks& start) const {
+  timer()->register_gc_concurrent_start(name(), start);
+
+  LogTarget(Debug, gc, phases, start) log;
+  log_start(log);
+}
+
+void XStatPhaseConcurrent::register_end(const Ticks& start, const Ticks& end) const {
+  if (XAbort::should_abort()) {
+    return;
+  }
+
+  timer()->register_gc_concurrent_end(end);
+
+  const Tickspan duration = end - start;
+  XStatSample(_sampler, duration.value());
+
+  LogTarget(Info, gc, phases) log;
+  log_end(log, duration);
+}
+
+XStatSubPhase::XStatSubPhase(const char* name) :
+    XStatPhase("Subphase", name) {}
+
+void XStatSubPhase::register_start(const Ticks& start) const {
+  if (XThread::is_worker()) {
+    LogTarget(Trace, gc, phases, start) log;
+    log_start(log, true /* thread */);
+  } else {
+    LogTarget(Debug, gc, phases, start) log;
+    log_start(log, false /* thread */);
+  }
+}
+
+void XStatSubPhase::register_end(const Ticks& start, const Ticks& end) const {
+  if (XAbort::should_abort()) {
+    return;
+  }
+
+  XTracer::tracer()->report_thread_phase(name(), start, end);
+
+  const Tickspan duration = end - start;
+  XStatSample(_sampler, duration.value());
+
+  if (XThread::is_worker()) {
+    LogTarget(Trace, gc, phases) log;
+    log_end(log, duration, true /* thread */);
+  } else {
+    LogTarget(Debug, gc, phases) log;
+    log_end(log, duration, false /* thread */);
+  }
+}
+
+XStatCriticalPhase::XStatCriticalPhase(const char* name, bool verbose) :
+    XStatPhase("Critical", name),
+    _counter("Critical", name, XStatUnitOpsPerSecond),
+    _verbose(verbose) {}
+
+void XStatCriticalPhase::register_start(const Ticks& start) const {
+  // This is called from sensitive contexts, for example before an allocation stall
+  // has been resolved. This means we must not access any oops in here since that
+  // could lead to infinite recursion. Without access to the thread name we can't
+  // really log anything useful here.
+}
+
+void XStatCriticalPhase::register_end(const Ticks& start, const Ticks& end) const {
+  XTracer::tracer()->report_thread_phase(name(), start, end);
+
+  const Tickspan duration = end - start;
+  XStatSample(_sampler, duration.value());
+  XStatInc(_counter);
+
+  if (_verbose) {
+    LogTarget(Info, gc) log;
+    log_end(log, duration, true /* thread */);
+  } else {
+    LogTarget(Debug, gc) log;
+    log_end(log, duration, true /* thread */);
+  }
+}
+
+//
+// Stat timer
+//
+THREAD_LOCAL uint32_t XStatTimerDisable::_active = 0;
+
+//
+// Stat sample/inc
+//
+void XStatSample(const XStatSampler& sampler, uint64_t value) {
+  XStatSamplerData* const cpu_data = sampler.get();
+  Atomic::add(&cpu_data->_nsamples, 1u);
+  Atomic::add(&cpu_data->_sum, value);
+
+  uint64_t max = cpu_data->_max;
+  for (;;) {
+    if (max >= value) {
+      // Not max
+      break;
+    }
+
+    const uint64_t new_max = value;
+    const uint64_t prev_max = Atomic::cmpxchg(&cpu_data->_max, max, new_max);
+    if (prev_max == max) {
+      // Success
+      break;
+    }
+
+    // Retry
+    max = prev_max;
+  }
+
+  XTracer::tracer()->report_stat_sampler(sampler, value);
+}
+
+void XStatInc(const XStatCounter& counter, uint64_t increment) {
+  XStatCounterData* const cpu_data = counter.get();
+  const uint64_t value = Atomic::add(&cpu_data->_counter, increment);
+
+  XTracer::tracer()->report_stat_counter(counter, increment, value);
+}
+
+void XStatInc(const XStatUnsampledCounter& counter, uint64_t increment) {
+  XStatCounterData* const cpu_data = counter.get();
+  Atomic::add(&cpu_data->_counter, increment);
+}
+
+//
+// Stat allocation rate
+//
+const XStatUnsampledCounter XStatAllocRate::_counter("Allocation Rate");
+TruncatedSeq                XStatAllocRate::_samples(XStatAllocRate::sample_hz);
+TruncatedSeq                XStatAllocRate::_rate(XStatAllocRate::sample_hz);
+
+const XStatUnsampledCounter& XStatAllocRate::counter() {
+  return _counter;
+}
+
+uint64_t XStatAllocRate::sample_and_reset() {
+  const XStatCounterData bytes_per_sample = _counter.collect_and_reset();
+  _samples.add(bytes_per_sample._counter);
+
+  const uint64_t bytes_per_second = _samples.sum();
+  _rate.add(bytes_per_second);
+
+  return bytes_per_second;
+}
+
+double XStatAllocRate::predict() {
+  return _rate.predict_next();
+}
+
+double XStatAllocRate::avg() {
+  return _rate.avg();
+}
+
+double XStatAllocRate::sd() {
+  return _rate.sd();
+}
+
+//
+// Stat thread
+//
+XStat::XStat() :
+    _metronome(sample_hz) {
+  set_name("XStat");
+  create_and_start();
+}
+
+void XStat::sample_and_collect(XStatSamplerHistory* history) const {
+  // Sample counters
+  for (const XStatCounter* counter = XStatCounter::first(); counter != NULL; counter = counter->next()) {
+    counter->sample_and_reset();
+  }
+
+  // Collect samples
+  for (const XStatSampler* sampler = XStatSampler::first(); sampler != NULL; sampler = sampler->next()) {
+    XStatSamplerHistory& sampler_history = history[sampler->id()];
+    sampler_history.add(sampler->collect_and_reset());
+  }
+}
+
+bool XStat::should_print(LogTargetHandle log) const {
+  static uint64_t print_at = ZStatisticsInterval;
+  const uint64_t now = os::elapsedTime();
+
+  if (now < print_at) {
+    return false;
+  }
+
+  print_at = ((now / ZStatisticsInterval) * ZStatisticsInterval) + ZStatisticsInterval;
+
+  return log.is_enabled();
+}
+
+void XStat::print(LogTargetHandle log, const XStatSamplerHistory* history) const {
+  // Print
+  log.print("=== Garbage Collection Statistics =======================================================================================================================");
+  log.print("                                                             Last 10s              Last 10m              Last 10h                Total");
+  log.print("                                                             Avg / Max             Avg / Max             Avg / Max             Avg / Max");
+
+  for (const XStatSampler* sampler = XStatSampler::first(); sampler != NULL; sampler = sampler->next()) {
+    const XStatSamplerHistory& sampler_history = history[sampler->id()];
+    const XStatUnitPrinter printer = sampler->printer();
+    printer(log, *sampler, sampler_history);
+  }
+
+  log.print("=========================================================================================================================================================");
+}
+
+void XStat::run_service() {
+  XStatSamplerHistory* const history = new XStatSamplerHistory[XStatSampler::count()];
+  LogTarget(Info, gc, stats) log;
+
+  XStatSampler::sort();
+
+  // Main loop
+  while (_metronome.wait_for_tick()) {
+    sample_and_collect(history);
+    if (should_print(log)) {
+      print(log, history);
+    }
+  }
+
+  delete [] history;
+}
+
+void XStat::stop_service() {
+  _metronome.stop();
+}
+
+//
+// Stat table
+//
+class XStatTablePrinter {
+private:
+  static const size_t _buffer_size = 256;
+
+  const size_t _column0_width;
+  const size_t _columnN_width;
+  char         _buffer[_buffer_size];
+
+public:
+  class XColumn {
+  private:
+    char* const  _buffer;
+    const size_t _position;
+    const size_t _width;
+    const size_t _width_next;
+
+    XColumn next() const {
+      // Insert space between columns
+      _buffer[_position + _width] = ' ';
+      return XColumn(_buffer, _position + _width + 1, _width_next, _width_next);
+    }
+
+    size_t print(size_t position, const char* fmt, va_list va) {
+      const int res = jio_vsnprintf(_buffer + position, _buffer_size - position, fmt, va);
+      if (res < 0) {
+        return 0;
+      }
+
+      return (size_t)res;
+    }
+
+  public:
+    XColumn(char* buffer, size_t position, size_t width, size_t width_next) :
+        _buffer(buffer),
+        _position(position),
+        _width(width),
+        _width_next(width_next) {}
+
+    XColumn left(const char* fmt, ...) ATTRIBUTE_PRINTF(2, 3) {
+      va_list va;
+
+      va_start(va, fmt);
+      const size_t written = print(_position, fmt, va);
+      va_end(va);
+
+      if (written < _width) {
+        // Fill empty space
+        memset(_buffer + _position + written, ' ', _width - written);
+      }
+
+      return next();
+    }
+
+    XColumn right(const char* fmt, ...) ATTRIBUTE_PRINTF(2, 3) {
+      va_list va;
+
+      va_start(va, fmt);
+      const size_t written = print(_position, fmt, va);
+      va_end(va);
+
+      if (written > _width) {
+        // Line too long
+        return fill('?');
+      }
+
+      if (written < _width) {
+        // Short line, move all to right
+        memmove(_buffer + _position + _width - written, _buffer + _position, written);
+
+        // Fill empty space
+        memset(_buffer + _position, ' ', _width - written);
+      }
+
+      return next();
+    }
+
+    XColumn center(const char* fmt, ...) ATTRIBUTE_PRINTF(2, 3) {
+      va_list va;
+
+      va_start(va, fmt);
+      const size_t written = print(_position, fmt, va);
+      va_end(va);
+
+      if (written > _width) {
+        // Line too long
+        return fill('?');
+      }
+
+      if (written < _width) {
+        // Short line, move all to center
+        const size_t start_space = (_width - written) / 2;
+        const size_t end_space = _width - written - start_space;
+        memmove(_buffer + _position + start_space, _buffer + _position, written);
+
+        // Fill empty spaces
+        memset(_buffer + _position, ' ', start_space);
+        memset(_buffer + _position + start_space + written, ' ', end_space);
+      }
+
+      return next();
+    }
+
+    XColumn fill(char filler = ' ') {
+      memset(_buffer + _position, filler, _width);
+      return next();
+    }
+
+    const char* end() {
+      _buffer[_position] = '\0';
+      return _buffer;
+    }
+  };
+
+public:
+  XStatTablePrinter(size_t column0_width, size_t columnN_width) :
+      _column0_width(column0_width),
+      _columnN_width(columnN_width) {}
+
+  XColumn operator()() {
+    return XColumn(_buffer, 0, _column0_width, _columnN_width);
+  }
+};
+
+//
+// Stat cycle
+//
+uint64_t  XStatCycle::_nwarmup_cycles = 0;
+Ticks     XStatCycle::_start_of_last;
+Ticks     XStatCycle::_end_of_last;
+NumberSeq XStatCycle::_serial_time(0.7 /* alpha */);
+NumberSeq XStatCycle::_parallelizable_time(0.7 /* alpha */);
+uint      XStatCycle::_last_active_workers = 0;
+
+void XStatCycle::at_start() {
+  _start_of_last = Ticks::now();
+}
+
+void XStatCycle::at_end(GCCause::Cause cause, uint active_workers) {
+  _end_of_last = Ticks::now();
+
+  if (cause == GCCause::_z_warmup) {
+    _nwarmup_cycles++;
+  }
+
+  _last_active_workers = active_workers;
+
+  // Calculate serial and parallelizable GC cycle times
+  const double duration = (_end_of_last - _start_of_last).seconds();
+  const double workers_duration = XStatWorkers::get_and_reset_duration();
+  const double serial_time = duration - workers_duration;
+  const double parallelizable_time = workers_duration * active_workers;
+  _serial_time.add(serial_time);
+  _parallelizable_time.add(parallelizable_time);
+}
+
+bool XStatCycle::is_warm() {
+  return _nwarmup_cycles >= 3;
+}
+
+uint64_t XStatCycle::nwarmup_cycles() {
+  return _nwarmup_cycles;
+}
+
+bool XStatCycle::is_time_trustable() {
+  // The times are considered trustable if we
+  // have completed at least one warmup cycle.
+  return _nwarmup_cycles > 0;
+}
+
+const AbsSeq& XStatCycle::serial_time() {
+  return _serial_time;
+}
+
+const AbsSeq& XStatCycle::parallelizable_time() {
+  return _parallelizable_time;
+}
+
+uint XStatCycle::last_active_workers() {
+  return _last_active_workers;
+}
+
+double XStatCycle::time_since_last() {
+  if (_end_of_last.value() == 0) {
+    // No end recorded yet, return time since VM start
+    return os::elapsedTime();
+  }
+
+  const Ticks now = Ticks::now();
+  const Tickspan time_since_last = now - _end_of_last;
+  return time_since_last.seconds();
+}
+
+//
+// Stat workers
+//
+Ticks XStatWorkers::_start_of_last;
+Tickspan XStatWorkers::_accumulated_duration;
+
+void XStatWorkers::at_start() {
+  _start_of_last = Ticks::now();
+}
+
+void XStatWorkers::at_end() {
+  const Ticks now = Ticks::now();
+  const Tickspan duration = now - _start_of_last;
+  _accumulated_duration += duration;
+}
+
+double XStatWorkers::get_and_reset_duration() {
+  const double duration = _accumulated_duration.seconds();
+  const Ticks now = Ticks::now();
+  _accumulated_duration = now - now;
+  return duration;
+}
+
+//
+// Stat load
+//
+void XStatLoad::print() {
+  double loadavg[3] = {};
+  os::loadavg(loadavg, ARRAY_SIZE(loadavg));
+  log_info(gc, load)("Load: %.2f/%.2f/%.2f", loadavg[0], loadavg[1], loadavg[2]);
+}
+
+//
+// Stat mark
+//
+size_t XStatMark::_nstripes;
+size_t XStatMark::_nproactiveflush;
+size_t XStatMark::_nterminateflush;
+size_t XStatMark::_ntrycomplete;
+size_t XStatMark::_ncontinue;
+size_t XStatMark::_mark_stack_usage;
+
+void XStatMark::set_at_mark_start(size_t nstripes) {
+  _nstripes = nstripes;
+}
+
+void XStatMark::set_at_mark_end(size_t nproactiveflush,
+                                size_t nterminateflush,
+                                size_t ntrycomplete,
+                                size_t ncontinue) {
+  _nproactiveflush = nproactiveflush;
+  _nterminateflush = nterminateflush;
+  _ntrycomplete = ntrycomplete;
+  _ncontinue = ncontinue;
+}
+
+void XStatMark::set_at_mark_free(size_t mark_stack_usage) {
+  _mark_stack_usage = mark_stack_usage;
+}
+
+void XStatMark::print() {
+  log_info(gc, marking)("Mark: "
+                        SIZE_FORMAT " stripe(s), "
+                        SIZE_FORMAT " proactive flush(es), "
+                        SIZE_FORMAT " terminate flush(es), "
+                        SIZE_FORMAT " completion(s), "
+                        SIZE_FORMAT " continuation(s) ",
+                        _nstripes,
+                        _nproactiveflush,
+                        _nterminateflush,
+                        _ntrycomplete,
+                        _ncontinue);
+
+  log_info(gc, marking)("Mark Stack Usage: " SIZE_FORMAT "M", _mark_stack_usage / M);
+}
+
+//
+// Stat relocation
+//
+XRelocationSetSelectorStats XStatRelocation::_selector_stats;
+size_t                      XStatRelocation::_forwarding_usage;
+size_t                      XStatRelocation::_small_in_place_count;
+size_t                      XStatRelocation::_medium_in_place_count;
+
+void XStatRelocation::set_at_select_relocation_set(const XRelocationSetSelectorStats& selector_stats) {
+  _selector_stats = selector_stats;
+}
+
+void XStatRelocation::set_at_install_relocation_set(size_t forwarding_usage) {
+  _forwarding_usage = forwarding_usage;
+}
+
+void XStatRelocation::set_at_relocate_end(size_t small_in_place_count, size_t medium_in_place_count) {
+  _small_in_place_count = small_in_place_count;
+  _medium_in_place_count = medium_in_place_count;
+}
+
+void XStatRelocation::print(const char* name,
+                            const XRelocationSetSelectorGroupStats& selector_group,
+                            size_t in_place_count) {
+  log_info(gc, reloc)("%s Pages: " SIZE_FORMAT " / " SIZE_FORMAT "M, Empty: " SIZE_FORMAT "M, "
+                      "Relocated: " SIZE_FORMAT "M, In-Place: " SIZE_FORMAT,
+                      name,
+                      selector_group.npages_candidates(),
+                      selector_group.total() / M,
+                      selector_group.empty() / M,
+                      selector_group.relocate() / M,
+                      in_place_count);
+}
+
+void XStatRelocation::print() {
+  print("Small", _selector_stats.small(), _small_in_place_count);
+  if (XPageSizeMedium != 0) {
+    print("Medium", _selector_stats.medium(), _medium_in_place_count);
+  }
+  print("Large", _selector_stats.large(), 0 /* in_place_count */);
+
+  log_info(gc, reloc)("Forwarding Usage: " SIZE_FORMAT "M", _forwarding_usage / M);
+}
+
+//
+// Stat nmethods
+//
+void XStatNMethods::print() {
+  log_info(gc, nmethod)("NMethods: " SIZE_FORMAT " registered, " SIZE_FORMAT " unregistered",
+                        XNMethodTable::registered_nmethods(),
+                        XNMethodTable::unregistered_nmethods());
+}
+
+//
+// Stat metaspace
+//
+void XStatMetaspace::print() {
+  MetaspaceCombinedStats stats = MetaspaceUtils::get_combined_statistics();
+  log_info(gc, metaspace)("Metaspace: "
+                          SIZE_FORMAT "M used, "
+                          SIZE_FORMAT "M committed, " SIZE_FORMAT "M reserved",
+                          stats.used() / M,
+                          stats.committed() / M,
+                          stats.reserved() / M);
+}
+
+//
+// Stat references
+//
+XStatReferences::XCount XStatReferences::_soft;
+XStatReferences::XCount XStatReferences::_weak;
+XStatReferences::XCount XStatReferences::_final;
+XStatReferences::XCount XStatReferences::_phantom;
+
+void XStatReferences::set(XCount* count, size_t encountered, size_t discovered, size_t enqueued) {
+  count->encountered = encountered;
+  count->discovered = discovered;
+  count->enqueued = enqueued;
+}
+
+void XStatReferences::set_soft(size_t encountered, size_t discovered, size_t enqueued) {
+  set(&_soft, encountered, discovered, enqueued);
+}
+
+void XStatReferences::set_weak(size_t encountered, size_t discovered, size_t enqueued) {
+  set(&_weak, encountered, discovered, enqueued);
+}
+
+void XStatReferences::set_final(size_t encountered, size_t discovered, size_t enqueued) {
+  set(&_final, encountered, discovered, enqueued);
+}
+
+void XStatReferences::set_phantom(size_t encountered, size_t discovered, size_t enqueued) {
+  set(&_phantom, encountered, discovered, enqueued);
+}
+
+void XStatReferences::print(const char* name, const XStatReferences::XCount& ref) {
+  log_info(gc, ref)("%s: "
+                    SIZE_FORMAT " encountered, "
+                    SIZE_FORMAT " discovered, "
+                    SIZE_FORMAT " enqueued",
+                    name,
+                    ref.encountered,
+                    ref.discovered,
+                    ref.enqueued);
+}
+
+void XStatReferences::print() {
+  print("Soft", _soft);
+  print("Weak", _weak);
+  print("Final", _final);
+  print("Phantom", _phantom);
+}
+
+//
+// Stat heap
+//
+XStatHeap::XAtInitialize XStatHeap::_at_initialize;
+XStatHeap::XAtMarkStart XStatHeap::_at_mark_start;
+XStatHeap::XAtMarkEnd XStatHeap::_at_mark_end;
+XStatHeap::XAtRelocateStart XStatHeap::_at_relocate_start;
+XStatHeap::XAtRelocateEnd XStatHeap::_at_relocate_end;
+
+size_t XStatHeap::capacity_high() {
+  return MAX4(_at_mark_start.capacity,
+              _at_mark_end.capacity,
+              _at_relocate_start.capacity,
+              _at_relocate_end.capacity);
+}
+
+size_t XStatHeap::capacity_low() {
+  return MIN4(_at_mark_start.capacity,
+              _at_mark_end.capacity,
+              _at_relocate_start.capacity,
+              _at_relocate_end.capacity);
+}
+
+size_t XStatHeap::free(size_t used) {
+  return _at_initialize.max_capacity - used;
+}
+
+size_t XStatHeap::allocated(size_t used, size_t reclaimed) {
+  // The amount of allocated memory between point A and B is used(B) - used(A).
+  // However, we might also have reclaimed memory between point A and B. This
+  // means the current amount of used memory must be incremented by the amount
+  // reclaimed, so that used(B) represents the amount of used memory we would
+  // have had if we had not reclaimed anything.
+  return (used + reclaimed) - _at_mark_start.used;
+}
+
+size_t XStatHeap::garbage(size_t reclaimed) {
+  return _at_mark_end.garbage - reclaimed;
+}
+
+void XStatHeap::set_at_initialize(const XPageAllocatorStats& stats) {
+  _at_initialize.min_capacity = stats.min_capacity();
+  _at_initialize.max_capacity = stats.max_capacity();
+}
+
+void XStatHeap::set_at_mark_start(const XPageAllocatorStats& stats) {
+  _at_mark_start.soft_max_capacity = stats.soft_max_capacity();
+  _at_mark_start.capacity = stats.capacity();
+  _at_mark_start.free = free(stats.used());
+  _at_mark_start.used = stats.used();
+}
+
+void XStatHeap::set_at_mark_end(const XPageAllocatorStats& stats) {
+  _at_mark_end.capacity = stats.capacity();
+  _at_mark_end.free = free(stats.used());
+  _at_mark_end.used = stats.used();
+  _at_mark_end.allocated = allocated(stats.used(), 0 /* reclaimed */);
+}
+
+void XStatHeap::set_at_select_relocation_set(const XRelocationSetSelectorStats& stats) {
+  const size_t live = stats.small().live() + stats.medium().live() + stats.large().live();
+  _at_mark_end.live = live;
+  _at_mark_end.garbage = _at_mark_start.used - live;
+}
+
+void XStatHeap::set_at_relocate_start(const XPageAllocatorStats& stats) {
+  _at_relocate_start.capacity = stats.capacity();
+  _at_relocate_start.free = free(stats.used());
+  _at_relocate_start.used = stats.used();
+  _at_relocate_start.allocated = allocated(stats.used(), stats.reclaimed());
+  _at_relocate_start.garbage = garbage(stats.reclaimed());
+  _at_relocate_start.reclaimed = stats.reclaimed();
+}
+
+void XStatHeap::set_at_relocate_end(const XPageAllocatorStats& stats, size_t non_worker_relocated) {
+  const size_t reclaimed = stats.reclaimed() - MIN2(non_worker_relocated, stats.reclaimed());
+
+  _at_relocate_end.capacity = stats.capacity();
+  _at_relocate_end.capacity_high = capacity_high();
+  _at_relocate_end.capacity_low = capacity_low();
+  _at_relocate_end.free = free(stats.used());
+  _at_relocate_end.free_high = free(stats.used_low());
+  _at_relocate_end.free_low = free(stats.used_high());
+  _at_relocate_end.used = stats.used();
+  _at_relocate_end.used_high = stats.used_high();
+  _at_relocate_end.used_low = stats.used_low();
+  _at_relocate_end.allocated = allocated(stats.used(), reclaimed);
+  _at_relocate_end.garbage = garbage(reclaimed);
+  _at_relocate_end.reclaimed = reclaimed;
+}
+
+size_t XStatHeap::max_capacity() {
+  return _at_initialize.max_capacity;
+}
+
+size_t XStatHeap::used_at_mark_start() {
+  return _at_mark_start.used;
+}
+
+size_t XStatHeap::used_at_relocate_end() {
+  return _at_relocate_end.used;
+}
+
+void XStatHeap::print() {
+  log_info(gc, heap)("Min Capacity: "
+                     XSIZE_FMT, XSIZE_ARGS(_at_initialize.min_capacity));
+  log_info(gc, heap)("Max Capacity: "
+                     XSIZE_FMT, XSIZE_ARGS(_at_initialize.max_capacity));
+  log_info(gc, heap)("Soft Max Capacity: "
+                     XSIZE_FMT, XSIZE_ARGS(_at_mark_start.soft_max_capacity));
+
+  XStatTablePrinter table(10, 18);
+  log_info(gc, heap)("%s", table()
+                     .fill()
+                     .center("Mark Start")
+                     .center("Mark End")
+                     .center("Relocate Start")
+                     .center("Relocate End")
+                     .center("High")
+                     .center("Low")
+                     .end());
+  log_info(gc, heap)("%s", table()
+                     .right("Capacity:")
+                     .left(XTABLE_ARGS(_at_mark_start.capacity))
+                     .left(XTABLE_ARGS(_at_mark_end.capacity))
+                     .left(XTABLE_ARGS(_at_relocate_start.capacity))
+                     .left(XTABLE_ARGS(_at_relocate_end.capacity))
+                     .left(XTABLE_ARGS(_at_relocate_end.capacity_high))
+                     .left(XTABLE_ARGS(_at_relocate_end.capacity_low))
+                     .end());
+  log_info(gc, heap)("%s", table()
+                     .right("Free:")
+                     .left(XTABLE_ARGS(_at_mark_start.free))
+                     .left(XTABLE_ARGS(_at_mark_end.free))
+                     .left(XTABLE_ARGS(_at_relocate_start.free))
+                     .left(XTABLE_ARGS(_at_relocate_end.free))
+                     .left(XTABLE_ARGS(_at_relocate_end.free_high))
+                     .left(XTABLE_ARGS(_at_relocate_end.free_low))
+                     .end());
+  log_info(gc, heap)("%s", table()
+                     .right("Used:")
+                     .left(XTABLE_ARGS(_at_mark_start.used))
+                     .left(XTABLE_ARGS(_at_mark_end.used))
+                     .left(XTABLE_ARGS(_at_relocate_start.used))
+                     .left(XTABLE_ARGS(_at_relocate_end.used))
+                     .left(XTABLE_ARGS(_at_relocate_end.used_high))
+                     .left(XTABLE_ARGS(_at_relocate_end.used_low))
+                     .end());
+  log_info(gc, heap)("%s", table()
+                     .right("Live:")
+                     .left(XTABLE_ARGS_NA)
+                     .left(XTABLE_ARGS(_at_mark_end.live))
+                     .left(XTABLE_ARGS(_at_mark_end.live /* Same as at mark end */))
+                     .left(XTABLE_ARGS(_at_mark_end.live /* Same as at mark end */))
+                     .left(XTABLE_ARGS_NA)
+                     .left(XTABLE_ARGS_NA)
+                     .end());
+  log_info(gc, heap)("%s", table()
+                     .right("Allocated:")
+                     .left(XTABLE_ARGS_NA)
+                     .left(XTABLE_ARGS(_at_mark_end.allocated))
+                     .left(XTABLE_ARGS(_at_relocate_start.allocated))
+                     .left(XTABLE_ARGS(_at_relocate_end.allocated))
+                     .left(XTABLE_ARGS_NA)
+                     .left(XTABLE_ARGS_NA)
+                     .end());
+  log_info(gc, heap)("%s", table()
+                     .right("Garbage:")
+                     .left(XTABLE_ARGS_NA)
+                     .left(XTABLE_ARGS(_at_mark_end.garbage))
+                     .left(XTABLE_ARGS(_at_relocate_start.garbage))
+                     .left(XTABLE_ARGS(_at_relocate_end.garbage))
+                     .left(XTABLE_ARGS_NA)
+                     .left(XTABLE_ARGS_NA)
+                     .end());
+  log_info(gc, heap)("%s", table()
+                     .right("Reclaimed:")
+                     .left(XTABLE_ARGS_NA)
+                     .left(XTABLE_ARGS_NA)
+                     .left(XTABLE_ARGS(_at_relocate_start.reclaimed))
+                     .left(XTABLE_ARGS(_at_relocate_end.reclaimed))
+                     .left(XTABLE_ARGS_NA)
+                     .left(XTABLE_ARGS_NA)
+                     .end());
+}
diff --git a/src/hotspot/share/gc/x/xStat.hpp b/src/hotspot/share/gc/x/xStat.hpp
new file mode 100644
index 00000000000..1ecaf9df492
--- /dev/null
+++ b/src/hotspot/share/gc/x/xStat.hpp
@@ -0,0 +1,578 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XSTAT_HPP
+#define SHARE_GC_X_XSTAT_HPP
+
+#include "gc/shared/concurrentGCThread.hpp"
+#include "gc/shared/gcCause.hpp"
+#include "gc/shared/gcTimer.hpp"
+#include "gc/x/xMetronome.hpp"
+#include "logging/logHandle.hpp"
+#include "memory/allocation.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/numberSeq.hpp"
+#include "utilities/ticks.hpp"
+
+class XPage;
+class XPageAllocatorStats;
+class XRelocationSetSelectorGroupStats;
+class XRelocationSetSelectorStats;
+class XStatSampler;
+class XStatSamplerHistory;
+struct XStatCounterData;
+struct XStatSamplerData;
+
+//
+// Stat unit printers
+//
+typedef void (*XStatUnitPrinter)(LogTargetHandle log, const XStatSampler&, const XStatSamplerHistory&);
+
+void XStatUnitTime(LogTargetHandle log, const XStatSampler& sampler, const XStatSamplerHistory& history);
+void XStatUnitBytes(LogTargetHandle log, const XStatSampler& sampler, const XStatSamplerHistory& history);
+void XStatUnitThreads(LogTargetHandle log, const XStatSampler& sampler, const XStatSamplerHistory& history);
+void XStatUnitBytesPerSecond(LogTargetHandle log, const XStatSampler& sampler, const XStatSamplerHistory& history);
+void XStatUnitOpsPerSecond(LogTargetHandle log, const XStatSampler& sampler, const XStatSamplerHistory& history);
+
+//
+// Stat value
+//
+class XStatValue {
+private:
+  static uintptr_t _base;
+  static uint32_t  _cpu_offset;
+
+  const char* const _group;
+  const char* const _name;
+  const uint32_t    _id;
+  const uint32_t    _offset;
+
+protected:
+  XStatValue(const char* group,
+             const char* name,
+             uint32_t id,
+             uint32_t size);
+
+  template <typename T> T* get_cpu_local(uint32_t cpu) const;
+
+public:
+  static void initialize();
+
+  const char* group() const;
+  const char* name() const;
+  uint32_t id() const;
+};
+
+//
+// Stat iterable value
+//
+template <typename T>
+class XStatIterableValue : public XStatValue {
+private:
+  static uint32_t _count;
+  static T*       _first;
+
+  T* _next;
+
+  T* insert() const;
+
+protected:
+  XStatIterableValue(const char* group,
+                     const char* name,
+                     uint32_t size);
+
+public:
+  static void sort();
+
+  static uint32_t count() {
+    return _count;
+  }
+
+  static T* first() {
+    return _first;
+  }
+
+  T* next() const {
+    return _next;
+  }
+};
+
+template <typename T> uint32_t XStatIterableValue<T>::_count = 0;
+template <typename T> T*       XStatIterableValue<T>::_first = NULL;
+
+//
+// Stat sampler
+//
+class XStatSampler : public XStatIterableValue<XStatSampler> {
+private:
+  const XStatUnitPrinter _printer;
+
+public:
+  XStatSampler(const char* group,
+               const char* name,
+               XStatUnitPrinter printer);
+
+  XStatSamplerData* get() const;
+  XStatSamplerData collect_and_reset() const;
+
+  XStatUnitPrinter printer() const;
+};
+
+//
+// Stat counter
+//
+class XStatCounter : public XStatIterableValue<XStatCounter> {
+private:
+  const XStatSampler _sampler;
+
+public:
+  XStatCounter(const char* group,
+               const char* name,
+               XStatUnitPrinter printer);
+
+  XStatCounterData* get() const;
+  void sample_and_reset() const;
+};
+
+//
+// Stat unsampled counter
+//
+class XStatUnsampledCounter : public XStatIterableValue<XStatUnsampledCounter> {
+public:
+  XStatUnsampledCounter(const char* name);
+
+  XStatCounterData* get() const;
+  XStatCounterData collect_and_reset() const;
+};
+
+//
+// Stat MMU (Minimum Mutator Utilization)
+//
+class XStatMMUPause {
+private:
+  double _start;
+  double _end;
+
+public:
+  XStatMMUPause();
+  XStatMMUPause(const Ticks& start, const Ticks& end);
+
+  double end() const;
+  double overlap(double start, double end) const;
+};
+
+class XStatMMU {
+private:
+  static size_t        _next;
+  static size_t        _npauses;
+  static XStatMMUPause _pauses[200]; // Record the last 200 pauses
+
+  static double _mmu_2ms;
+  static double _mmu_5ms;
+  static double _mmu_10ms;
+  static double _mmu_20ms;
+  static double _mmu_50ms;
+  static double _mmu_100ms;
+
+  static const XStatMMUPause& pause(size_t index);
+  static double calculate_mmu(double time_slice);
+
+public:
+  static void register_pause(const Ticks& start, const Ticks& end);
+
+  static void print();
+};
+
+//
+// Stat phases
+//
+class XStatPhase {
+private:
+  static ConcurrentGCTimer _timer;
+
+protected:
+  const XStatSampler _sampler;
+
+  XStatPhase(const char* group, const char* name);
+
+  void log_start(LogTargetHandle log, bool thread = false) const;
+  void log_end(LogTargetHandle log, const Tickspan& duration, bool thread = false) const;
+
+public:
+  static ConcurrentGCTimer* timer();
+
+  const char* name() const;
+
+  virtual void register_start(const Ticks& start) const = 0;
+  virtual void register_end(const Ticks& start, const Ticks& end) const = 0;
+};
+
+class XStatPhaseCycle : public XStatPhase {
+public:
+  XStatPhaseCycle(const char* name);
+
+  virtual void register_start(const Ticks& start) const;
+  virtual void register_end(const Ticks& start, const Ticks& end) const;
+};
+
+class XStatPhasePause : public XStatPhase {
+private:
+  static Tickspan _max; // Max pause time
+
+public:
+  XStatPhasePause(const char* name);
+
+  static const Tickspan& max();
+
+  virtual void register_start(const Ticks& start) const;
+  virtual void register_end(const Ticks& start, const Ticks& end) const;
+};
+
+class XStatPhaseConcurrent : public XStatPhase {
+public:
+  XStatPhaseConcurrent(const char* name);
+
+  virtual void register_start(const Ticks& start) const;
+  virtual void register_end(const Ticks& start, const Ticks& end) const;
+};
+
+class XStatSubPhase : public XStatPhase {
+public:
+  XStatSubPhase(const char* name);
+
+  virtual void register_start(const Ticks& start) const;
+  virtual void register_end(const Ticks& start, const Ticks& end) const;
+};
+
+class XStatCriticalPhase : public XStatPhase {
+private:
+  const XStatCounter _counter;
+  const bool         _verbose;
+
+public:
+  XStatCriticalPhase(const char* name, bool verbose = true);
+
+  virtual void register_start(const Ticks& start) const;
+  virtual void register_end(const Ticks& start, const Ticks& end) const;
+};
+
+//
+// Stat timer
+//
+class XStatTimerDisable : public StackObj {
+private:
+  static THREAD_LOCAL uint32_t _active;
+
+public:
+  XStatTimerDisable() {
+    _active++;
+  }
+
+  ~XStatTimerDisable() {
+    _active--;
+  }
+
+  static bool is_active() {
+    return _active > 0;
+  }
+};
+
+class XStatTimer : public StackObj {
+private:
+  const bool        _enabled;
+  const XStatPhase& _phase;
+  const Ticks       _start;
+
+public:
+  XStatTimer(const XStatPhase& phase) :
+      _enabled(!XStatTimerDisable::is_active()),
+      _phase(phase),
+      _start(Ticks::now()) {
+    if (_enabled) {
+      _phase.register_start(_start);
+    }
+  }
+
+  ~XStatTimer() {
+    if (_enabled) {
+      const Ticks end = Ticks::now();
+      _phase.register_end(_start, end);
+    }
+  }
+};
+
+//
+// Stat sample/increment
+//
+void XStatSample(const XStatSampler& sampler, uint64_t value);
+void XStatInc(const XStatCounter& counter, uint64_t increment = 1);
+void XStatInc(const XStatUnsampledCounter& counter, uint64_t increment = 1);
+
+//
+// Stat allocation rate
+//
+class XStatAllocRate : public AllStatic {
+private:
+  static const XStatUnsampledCounter _counter;
+  static TruncatedSeq                _samples;
+  static TruncatedSeq                _rate;
+
+public:
+  static const uint64_t sample_hz = 10;
+
+  static const XStatUnsampledCounter& counter();
+  static uint64_t sample_and_reset();
+
+  static double predict();
+  static double avg();
+  static double sd();
+};
+
+//
+// Stat thread
+//
+class XStat : public ConcurrentGCThread {
+private:
+  static const uint64_t sample_hz = 1;
+
+  XMetronome _metronome;
+
+  void sample_and_collect(XStatSamplerHistory* history) const;
+  bool should_print(LogTargetHandle log) const;
+  void print(LogTargetHandle log, const XStatSamplerHistory* history) const;
+
+protected:
+  virtual void run_service();
+  virtual void stop_service();
+
+public:
+  XStat();
+};
+
+//
+// Stat cycle
+//
+class XStatCycle : public AllStatic {
+private:
+  static uint64_t  _nwarmup_cycles;
+  static Ticks     _start_of_last;
+  static Ticks     _end_of_last;
+  static NumberSeq _serial_time;
+  static NumberSeq _parallelizable_time;
+  static uint      _last_active_workers;
+
+public:
+  static void at_start();
+  static void at_end(GCCause::Cause cause, uint active_workers);
+
+  static bool is_warm();
+  static uint64_t nwarmup_cycles();
+
+  static bool is_time_trustable();
+  static const AbsSeq& serial_time();
+  static const AbsSeq& parallelizable_time();
+
+  static uint last_active_workers();
+
+  static double time_since_last();
+};
+
+//
+// Stat workers
+//
+class XStatWorkers : public AllStatic {
+private:
+  static Ticks    _start_of_last;
+  static Tickspan _accumulated_duration;
+
+public:
+  static void at_start();
+  static void at_end();
+
+  static double get_and_reset_duration();
+};
+
+//
+// Stat load
+//
+class XStatLoad : public AllStatic {
+public:
+  static void print();
+};
+
+//
+// Stat mark
+//
+class XStatMark : public AllStatic {
+private:
+  static size_t _nstripes;
+  static size_t _nproactiveflush;
+  static size_t _nterminateflush;
+  static size_t _ntrycomplete;
+  static size_t _ncontinue;
+  static size_t _mark_stack_usage;
+
+public:
+  static void set_at_mark_start(size_t nstripes);
+  static void set_at_mark_end(size_t nproactiveflush,
+                              size_t nterminateflush,
+                              size_t ntrycomplete,
+                              size_t ncontinue);
+  static void set_at_mark_free(size_t mark_stack_usage);
+
+  static void print();
+};
+
+//
+// Stat relocation
+//
+class XStatRelocation : public AllStatic {
+private:
+  static XRelocationSetSelectorStats _selector_stats;
+  static size_t                      _forwarding_usage;
+  static size_t                      _small_in_place_count;
+  static size_t                      _medium_in_place_count;
+
+  static void print(const char* name,
+                    const XRelocationSetSelectorGroupStats& selector_group,
+                    size_t in_place_count);
+
+public:
+  static void set_at_select_relocation_set(const XRelocationSetSelectorStats& selector_stats);
+  static void set_at_install_relocation_set(size_t forwarding_usage);
+  static void set_at_relocate_end(size_t small_in_place_count, size_t medium_in_place_count);
+
+  static void print();
+};
+
+//
+// Stat nmethods
+//
+class XStatNMethods : public AllStatic {
+public:
+  static void print();
+};
+
+//
+// Stat metaspace
+//
+class XStatMetaspace : public AllStatic {
+public:
+  static void print();
+};
+
+//
+// Stat references
+//
+class XStatReferences : public AllStatic {
+private:
+  static struct XCount {
+    size_t encountered;
+    size_t discovered;
+    size_t enqueued;
+  } _soft, _weak, _final, _phantom;
+
+  static void set(XCount* count, size_t encountered, size_t discovered, size_t enqueued);
+  static void print(const char* name, const XCount& ref);
+
+public:
+  static void set_soft(size_t encountered, size_t discovered, size_t enqueued);
+  static void set_weak(size_t encountered, size_t discovered, size_t enqueued);
+  static void set_final(size_t encountered, size_t discovered, size_t enqueued);
+  static void set_phantom(size_t encountered, size_t discovered, size_t enqueued);
+
+  static void print();
+};
+
+//
+// Stat heap
+//
+class XStatHeap : public AllStatic {
+private:
+  static struct XAtInitialize {
+    size_t min_capacity;
+    size_t max_capacity;
+  } _at_initialize;
+
+  static struct XAtMarkStart {
+    size_t soft_max_capacity;
+    size_t capacity;
+    size_t free;
+    size_t used;
+  } _at_mark_start;
+
+  static struct XAtMarkEnd {
+    size_t capacity;
+    size_t free;
+    size_t used;
+    size_t live;
+    size_t allocated;
+    size_t garbage;
+  } _at_mark_end;
+
+  static struct XAtRelocateStart {
+    size_t capacity;
+    size_t free;
+    size_t used;
+    size_t allocated;
+    size_t garbage;
+    size_t reclaimed;
+  } _at_relocate_start;
+
+  static struct XAtRelocateEnd {
+    size_t capacity;
+    size_t capacity_high;
+    size_t capacity_low;
+    size_t free;
+    size_t free_high;
+    size_t free_low;
+    size_t used;
+    size_t used_high;
+    size_t used_low;
+    size_t allocated;
+    size_t garbage;
+    size_t reclaimed;
+  } _at_relocate_end;
+
+  static size_t capacity_high();
+  static size_t capacity_low();
+  static size_t free(size_t used);
+  static size_t allocated(size_t used, size_t reclaimed);
+  static size_t garbage(size_t reclaimed);
+
+public:
+  static void set_at_initialize(const XPageAllocatorStats& stats);
+  static void set_at_mark_start(const XPageAllocatorStats& stats);
+  static void set_at_mark_end(const XPageAllocatorStats& stats);
+  static void set_at_select_relocation_set(const XRelocationSetSelectorStats& stats);
+  static void set_at_relocate_start(const XPageAllocatorStats& stats);
+  static void set_at_relocate_end(const XPageAllocatorStats& stats, size_t non_worker_relocated);
+
+  static size_t max_capacity();
+  static size_t used_at_mark_start();
+  static size_t used_at_relocate_end();
+
+  static void print();
+};
+
+#endif // SHARE_GC_X_XSTAT_HPP
diff --git a/src/hotspot/share/gc/x/xTask.cpp b/src/hotspot/share/gc/x/xTask.cpp
new file mode 100644
index 00000000000..25f6d12f33d
--- /dev/null
+++ b/src/hotspot/share/gc/x/xTask.cpp
@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xTask.hpp"
+#include "gc/x/xThread.hpp"
+
+XTask::Task::Task(XTask* task, const char* name) :
+    WorkerTask(name),
+    _task(task) {}
+
+void XTask::Task::work(uint worker_id) {
+  XThread::set_worker_id(worker_id);
+  _task->work();
+  XThread::clear_worker_id();
+}
+
+XTask::XTask(const char* name) :
+    _worker_task(this, name) {}
+
+const char* XTask::name() const {
+  return _worker_task.name();
+}
+
+WorkerTask* XTask::worker_task() {
+  return &_worker_task;
+}
diff --git a/src/hotspot/share/gc/x/xTask.hpp b/src/hotspot/share/gc/x/xTask.hpp
new file mode 100644
index 00000000000..08adaed83e5
--- /dev/null
+++ b/src/hotspot/share/gc/x/xTask.hpp
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XTASK_HPP
+#define SHARE_GC_X_XTASK_HPP
+
+#include "gc/shared/workerThread.hpp"
+#include "memory/allocation.hpp"
+
+class XTask : public StackObj {
+private:
+  class Task : public WorkerTask {
+  private:
+    XTask* const _task;
+
+  public:
+    Task(XTask* task, const char* name);
+
+    virtual void work(uint worker_id);
+  };
+
+  Task _worker_task;
+
+public:
+  XTask(const char* name);
+
+  const char* name() const;
+  WorkerTask* worker_task();
+
+  virtual void work() = 0;
+};
+
+#endif // SHARE_GC_X_XTASK_HPP
diff --git a/src/hotspot/share/gc/x/xThread.cpp b/src/hotspot/share/gc/x/xThread.cpp
new file mode 100644
index 00000000000..fb9785690cf
--- /dev/null
+++ b/src/hotspot/share/gc/x/xThread.cpp
@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xThread.inline.hpp"
+#include "runtime/javaThread.hpp"
+#include "runtime/nonJavaThread.hpp"
+#include "utilities/debug.hpp"
+
+THREAD_LOCAL bool      XThread::_initialized;
+THREAD_LOCAL uintptr_t XThread::_id;
+THREAD_LOCAL bool      XThread::_is_vm;
+THREAD_LOCAL bool      XThread::_is_java;
+THREAD_LOCAL bool      XThread::_is_worker;
+THREAD_LOCAL uint      XThread::_worker_id;
+
+void XThread::initialize() {
+  assert(!_initialized, "Already initialized");
+  const Thread* const thread = Thread::current();
+  _initialized = true;
+  _id = (uintptr_t)thread;
+  _is_vm = thread->is_VM_thread();
+  _is_java = thread->is_Java_thread();
+  _is_worker = false;
+  _worker_id = (uint)-1;
+}
+
+const char* XThread::name() {
+  const Thread* const thread = Thread::current();
+  if (thread->is_Named_thread()) {
+    const NamedThread* const named = (const NamedThread*)thread;
+    return named->name();
+  } else if (thread->is_Java_thread()) {
+    return "Java";
+  }
+
+  return "Unknown";
+}
+
+void XThread::set_worker() {
+  ensure_initialized();
+  _is_worker = true;
+}
+
+bool XThread::has_worker_id() {
+  return _initialized &&
+         _is_worker &&
+         _worker_id != (uint)-1;
+}
+
+void XThread::set_worker_id(uint worker_id) {
+  ensure_initialized();
+  assert(!has_worker_id(), "Worker id already initialized");
+  _worker_id = worker_id;
+}
+
+void XThread::clear_worker_id() {
+  assert(has_worker_id(), "Worker id not initialized");
+  _worker_id = (uint)-1;
+}
diff --git a/src/hotspot/share/gc/x/xThread.hpp b/src/hotspot/share/gc/x/xThread.hpp
new file mode 100644
index 00000000000..24df6ce1ca2
--- /dev/null
+++ b/src/hotspot/share/gc/x/xThread.hpp
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XTHREAD_HPP
+#define SHARE_GC_X_XTHREAD_HPP
+
+#include "memory/allStatic.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class XThread : public AllStatic {
+  friend class XTask;
+  friend class XWorkersInitializeTask;
+  friend class XRuntimeWorkersInitializeTask;
+
+private:
+  static THREAD_LOCAL bool      _initialized;
+  static THREAD_LOCAL uintptr_t _id;
+  static THREAD_LOCAL bool      _is_vm;
+  static THREAD_LOCAL bool      _is_java;
+  static THREAD_LOCAL bool      _is_worker;
+  static THREAD_LOCAL uint      _worker_id;
+
+  static void initialize();
+  static void ensure_initialized();
+
+  static void set_worker();
+
+  static bool has_worker_id();
+  static void set_worker_id(uint worker_id);
+  static void clear_worker_id();
+
+public:
+  static const char* name();
+  static uintptr_t id();
+  static bool is_vm();
+  static bool is_java();
+  static bool is_worker();
+  static uint worker_id();
+};
+
+#endif // SHARE_GC_X_XTHREAD_HPP
diff --git a/src/hotspot/share/gc/z/zThread.inline.hpp b/src/hotspot/share/gc/x/xThread.inline.hpp
similarity index 79%
rename from src/hotspot/share/gc/z/zThread.inline.hpp
rename to src/hotspot/share/gc/x/xThread.inline.hpp
index 1a88648fc75..eb6ff63e5f7 100644
--- a/src/hotspot/share/gc/z/zThread.inline.hpp
+++ b/src/hotspot/share/gc/x/xThread.inline.hpp
@@ -21,42 +21,42 @@
  * questions.
  */
 
-#ifndef SHARE_GC_Z_ZTHREAD_INLINE_HPP
-#define SHARE_GC_Z_ZTHREAD_INLINE_HPP
+#ifndef SHARE_GC_X_XTHREAD_INLINE_HPP
+#define SHARE_GC_X_XTHREAD_INLINE_HPP
 
-#include "gc/z/zThread.hpp"
+#include "gc/x/xThread.hpp"
 
 #include "utilities/debug.hpp"
 
-inline void ZThread::ensure_initialized() {
+inline void XThread::ensure_initialized() {
   if (!_initialized) {
     initialize();
   }
 }
 
-inline uintptr_t ZThread::id() {
+inline uintptr_t XThread::id() {
   ensure_initialized();
   return _id;
 }
 
-inline bool ZThread::is_vm() {
+inline bool XThread::is_vm() {
   ensure_initialized();
   return _is_vm;
 }
 
-inline bool ZThread::is_java() {
+inline bool XThread::is_java() {
   ensure_initialized();
   return _is_java;
 }
 
-inline bool ZThread::is_worker() {
+inline bool XThread::is_worker() {
   ensure_initialized();
   return _is_worker;
 }
 
-inline uint ZThread::worker_id() {
+inline uint XThread::worker_id() {
   assert(has_worker_id(), "Worker id not initialized");
   return _worker_id;
 }
 
-#endif // SHARE_GC_Z_ZTHREAD_INLINE_HPP
+#endif // SHARE_GC_X_XTHREAD_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xThreadLocalAllocBuffer.cpp b/src/hotspot/share/gc/x/xThreadLocalAllocBuffer.cpp
new file mode 100644
index 00000000000..594a7799d7d
--- /dev/null
+++ b/src/hotspot/share/gc/x/xThreadLocalAllocBuffer.cpp
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/tlab_globals.hpp"
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xStackWatermark.hpp"
+#include "gc/x/xThreadLocalAllocBuffer.hpp"
+#include "gc/x/xValue.inline.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/javaThread.hpp"
+#include "runtime/stackWatermarkSet.inline.hpp"
+
+XPerWorker<ThreadLocalAllocStats>* XThreadLocalAllocBuffer::_stats = NULL;
+
+void XThreadLocalAllocBuffer::initialize() {
+  if (UseTLAB) {
+    assert(_stats == NULL, "Already initialized");
+    _stats = new XPerWorker<ThreadLocalAllocStats>();
+    reset_statistics();
+  }
+}
+
+void XThreadLocalAllocBuffer::reset_statistics() {
+  if (UseTLAB) {
+    XPerWorkerIterator<ThreadLocalAllocStats> iter(_stats);
+    for (ThreadLocalAllocStats* stats; iter.next(&stats);) {
+      stats->reset();
+    }
+  }
+}
+
+void XThreadLocalAllocBuffer::publish_statistics() {
+  if (UseTLAB) {
+    ThreadLocalAllocStats total;
+
+    XPerWorkerIterator<ThreadLocalAllocStats> iter(_stats);
+    for (ThreadLocalAllocStats* stats; iter.next(&stats);) {
+      total.update(*stats);
+    }
+
+    total.publish();
+  }
+}
+
+static void fixup_address(HeapWord** p) {
+  *p = (HeapWord*)XAddress::good_or_null((uintptr_t)*p);
+}
+
+void XThreadLocalAllocBuffer::retire(JavaThread* thread, ThreadLocalAllocStats* stats) {
+  if (UseTLAB) {
+    stats->reset();
+    thread->tlab().addresses_do(fixup_address);
+    thread->tlab().retire(stats);
+    if (ResizeTLAB) {
+      thread->tlab().resize();
+    }
+  }
+}
+
+void XThreadLocalAllocBuffer::remap(JavaThread* thread) {
+  if (UseTLAB) {
+    thread->tlab().addresses_do(fixup_address);
+  }
+}
+
+void XThreadLocalAllocBuffer::update_stats(JavaThread* thread) {
+  if (UseTLAB) {
+    XStackWatermark* const watermark = StackWatermarkSet::get<XStackWatermark>(thread, StackWatermarkKind::gc);
+    _stats->addr()->update(watermark->stats());
+  }
+}
diff --git a/src/hotspot/share/gc/x/xThreadLocalAllocBuffer.hpp b/src/hotspot/share/gc/x/xThreadLocalAllocBuffer.hpp
new file mode 100644
index 00000000000..521f4da1909
--- /dev/null
+++ b/src/hotspot/share/gc/x/xThreadLocalAllocBuffer.hpp
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XTHREADLOCALALLOCBUFFER_HPP
+#define SHARE_GC_X_XTHREADLOCALALLOCBUFFER_HPP
+
+#include "gc/shared/threadLocalAllocBuffer.hpp"
+#include "gc/x/xValue.hpp"
+#include "memory/allStatic.hpp"
+
+class JavaThread;
+
+class XThreadLocalAllocBuffer : public AllStatic {
+private:
+  static XPerWorker<ThreadLocalAllocStats>* _stats;
+
+public:
+  static void initialize();
+
+  static void reset_statistics();
+  static void publish_statistics();
+
+  static void retire(JavaThread* thread, ThreadLocalAllocStats* stats);
+  static void remap(JavaThread* thread);
+  static void update_stats(JavaThread* thread);
+};
+
+#endif // SHARE_GC_X_XTHREADLOCALALLOCBUFFER_HPP
diff --git a/src/hotspot/share/gc/x/xThreadLocalData.hpp b/src/hotspot/share/gc/x/xThreadLocalData.hpp
new file mode 100644
index 00000000000..b4abaadd09c
--- /dev/null
+++ b/src/hotspot/share/gc/x/xThreadLocalData.hpp
@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XTHREADLOCALDATA_HPP
+#define SHARE_GC_X_XTHREADLOCALDATA_HPP
+
+#include "gc/x/xMarkStack.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "runtime/javaThread.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/sizes.hpp"
+
+class XThreadLocalData {
+private:
+  uintptr_t              _address_bad_mask;
+  XMarkThreadLocalStacks _stacks;
+  oop*                   _invisible_root;
+
+  XThreadLocalData() :
+      _address_bad_mask(0),
+      _stacks(),
+      _invisible_root(NULL) {}
+
+  static XThreadLocalData* data(Thread* thread) {
+    return thread->gc_data<XThreadLocalData>();
+  }
+
+public:
+  static void create(Thread* thread) {
+    new (data(thread)) XThreadLocalData();
+  }
+
+  static void destroy(Thread* thread) {
+    data(thread)->~XThreadLocalData();
+  }
+
+  static void set_address_bad_mask(Thread* thread, uintptr_t mask) {
+    data(thread)->_address_bad_mask = mask;
+  }
+
+  static XMarkThreadLocalStacks* stacks(Thread* thread) {
+    return &data(thread)->_stacks;
+  }
+
+  static void set_invisible_root(Thread* thread, oop* root) {
+    assert(data(thread)->_invisible_root == NULL, "Already set");
+    data(thread)->_invisible_root = root;
+  }
+
+  static void clear_invisible_root(Thread* thread) {
+    assert(data(thread)->_invisible_root != NULL, "Should be set");
+    data(thread)->_invisible_root = NULL;
+  }
+
+  template <typename T>
+  static void do_invisible_root(Thread* thread, T f) {
+    if (data(thread)->_invisible_root != NULL) {
+      f(data(thread)->_invisible_root);
+    }
+  }
+
+  static ByteSize address_bad_mask_offset() {
+    return Thread::gc_data_offset() + byte_offset_of(XThreadLocalData, _address_bad_mask);
+  }
+
+  static ByteSize nmethod_disarmed_offset() {
+    return address_bad_mask_offset() + in_ByteSize(XAddressBadMaskHighOrderBitsOffset);
+  }
+};
+
+#endif // SHARE_GC_X_XTHREADLOCALDATA_HPP
diff --git a/src/hotspot/share/gc/x/xTracer.cpp b/src/hotspot/share/gc/x/xTracer.cpp
new file mode 100644
index 00000000000..6db2e0bcc9a
--- /dev/null
+++ b/src/hotspot/share/gc/x/xTracer.cpp
@@ -0,0 +1,146 @@
+/*
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcId.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xTracer.hpp"
+#include "jfr/jfrEvents.hpp"
+#include "runtime/safepointVerifiers.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/macros.hpp"
+#if INCLUDE_JFR
+#include "jfr/metadata/jfrSerializer.hpp"
+#endif
+
+#if INCLUDE_JFR
+
+class XPageTypeConstant : public JfrSerializer {
+public:
+  virtual void serialize(JfrCheckpointWriter& writer) {
+    writer.write_count(3);
+    writer.write_key(XPageTypeSmall);
+    writer.write("Small");
+    writer.write_key(XPageTypeMedium);
+    writer.write("Medium");
+    writer.write_key(XPageTypeLarge);
+    writer.write("Large");
+  }
+};
+
+class XStatisticsCounterTypeConstant : public JfrSerializer {
+public:
+  virtual void serialize(JfrCheckpointWriter& writer) {
+    writer.write_count(XStatCounter::count());
+    for (XStatCounter* counter = XStatCounter::first(); counter != NULL; counter = counter->next()) {
+      writer.write_key(counter->id());
+      writer.write(counter->name());
+    }
+  }
+};
+
+class XStatisticsSamplerTypeConstant : public JfrSerializer {
+public:
+  virtual void serialize(JfrCheckpointWriter& writer) {
+    writer.write_count(XStatSampler::count());
+    for (XStatSampler* sampler = XStatSampler::first(); sampler != NULL; sampler = sampler->next()) {
+      writer.write_key(sampler->id());
+      writer.write(sampler->name());
+    }
+  }
+};
+
+static void register_jfr_type_serializers() {
+  JfrSerializer::register_serializer(TYPE_ZPAGETYPETYPE,
+                                     true /* permit_cache */,
+                                     new XPageTypeConstant());
+  JfrSerializer::register_serializer(TYPE_ZSTATISTICSCOUNTERTYPE,
+                                     true /* permit_cache */,
+                                     new XStatisticsCounterTypeConstant());
+  JfrSerializer::register_serializer(TYPE_ZSTATISTICSSAMPLERTYPE,
+                                     true /* permit_cache */,
+                                     new XStatisticsSamplerTypeConstant());
+}
+
+#endif // INCLUDE_JFR
+
+XTracer* XTracer::_tracer = NULL;
+
+XTracer::XTracer() :
+    GCTracer(Z) {}
+
+void XTracer::initialize() {
+  assert(_tracer == NULL, "Already initialized");
+  _tracer = new XTracer();
+  JFR_ONLY(register_jfr_type_serializers());
+}
+
+void XTracer::send_stat_counter(const XStatCounter& counter, uint64_t increment, uint64_t value) {
+  NoSafepointVerifier nsv;
+
+  EventZStatisticsCounter e;
+  if (e.should_commit()) {
+    e.set_id(counter.id());
+    e.set_increment(increment);
+    e.set_value(value);
+    e.commit();
+  }
+}
+
+void XTracer::send_stat_sampler(const XStatSampler& sampler, uint64_t value) {
+  NoSafepointVerifier nsv;
+
+  EventZStatisticsSampler e;
+  if (e.should_commit()) {
+    e.set_id(sampler.id());
+    e.set_value(value);
+    e.commit();
+  }
+}
+
+void XTracer::send_thread_phase(const char* name, const Ticks& start, const Ticks& end) {
+  NoSafepointVerifier nsv;
+
+  EventZThreadPhase e(UNTIMED);
+  if (e.should_commit()) {
+    e.set_gcId(GCId::current_or_undefined());
+    e.set_name(name);
+    e.set_starttime(start);
+    e.set_endtime(end);
+    e.commit();
+  }
+}
+
+void XTracer::send_thread_debug(const char* name, const Ticks& start, const Ticks& end) {
+  NoSafepointVerifier nsv;
+
+  EventZThreadDebug e(UNTIMED);
+  if (e.should_commit()) {
+    e.set_gcId(GCId::current_or_undefined());
+    e.set_name(name);
+    e.set_starttime(start);
+    e.set_endtime(end);
+    e.commit();
+  }
+}
diff --git a/src/hotspot/share/gc/x/xTracer.hpp b/src/hotspot/share/gc/x/xTracer.hpp
new file mode 100644
index 00000000000..d9219d79c51
--- /dev/null
+++ b/src/hotspot/share/gc/x/xTracer.hpp
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XTRACER_HPP
+#define SHARE_GC_X_XTRACER_HPP
+
+#include "gc/shared/gcTrace.hpp"
+
+class XStatCounter;
+class XStatPhase;
+class XStatSampler;
+
+class XTracer : public GCTracer, public CHeapObj<mtGC> {
+private:
+  static XTracer* _tracer;
+
+  XTracer();
+
+  void send_stat_counter(const XStatCounter& counter, uint64_t increment, uint64_t value);
+  void send_stat_sampler(const XStatSampler& sampler, uint64_t value);
+  void send_thread_phase(const char* name, const Ticks& start, const Ticks& end);
+  void send_thread_debug(const char* name, const Ticks& start, const Ticks& end);
+
+public:
+  static XTracer* tracer();
+  static void initialize();
+
+  void report_stat_counter(const XStatCounter& counter, uint64_t increment, uint64_t value);
+  void report_stat_sampler(const XStatSampler& sampler, uint64_t value);
+  void report_thread_phase(const char* name, const Ticks& start, const Ticks& end);
+  void report_thread_debug(const char* name, const Ticks& start, const Ticks& end);
+};
+
+// For temporary latency measurements during development and debugging
+class XTraceThreadDebug : public StackObj {
+private:
+  const Ticks       _start;
+  const char* const _name;
+
+public:
+  XTraceThreadDebug(const char* name);
+  ~XTraceThreadDebug();
+};
+
+#endif // SHARE_GC_X_XTRACER_HPP
diff --git a/src/hotspot/share/gc/x/xTracer.inline.hpp b/src/hotspot/share/gc/x/xTracer.inline.hpp
new file mode 100644
index 00000000000..22dd2e2b6fb
--- /dev/null
+++ b/src/hotspot/share/gc/x/xTracer.inline.hpp
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XTRACER_INLINE_HPP
+#define SHARE_GC_X_XTRACER_INLINE_HPP
+
+#include "gc/x/xTracer.hpp"
+
+#include "jfr/jfrEvents.hpp"
+
+inline XTracer* XTracer::tracer() {
+  return _tracer;
+}
+
+inline void XTracer::report_stat_counter(const XStatCounter& counter, uint64_t increment, uint64_t value) {
+  if (EventZStatisticsCounter::is_enabled()) {
+    send_stat_counter(counter, increment, value);
+  }
+}
+
+inline void XTracer::report_stat_sampler(const XStatSampler& sampler, uint64_t value) {
+  if (EventZStatisticsSampler::is_enabled()) {
+    send_stat_sampler(sampler, value);
+  }
+}
+
+inline void XTracer::report_thread_phase(const char* name, const Ticks& start, const Ticks& end) {
+  if (EventZThreadPhase::is_enabled()) {
+    send_thread_phase(name, start, end);
+  }
+}
+
+inline void XTracer::report_thread_debug(const char* name, const Ticks& start, const Ticks& end) {
+  if (EventZThreadDebug::is_enabled()) {
+    send_thread_debug(name, start, end);
+  }
+}
+
+inline XTraceThreadDebug::XTraceThreadDebug(const char* name) :
+    _start(Ticks::now()),
+    _name(name) {}
+
+inline XTraceThreadDebug::~XTraceThreadDebug() {
+  XTracer::tracer()->report_thread_debug(_name, _start, Ticks::now());
+}
+
+#endif // SHARE_GC_X_XTRACER_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xUncommitter.cpp b/src/hotspot/share/gc/x/xUncommitter.cpp
new file mode 100644
index 00000000000..ffd57b8c2a8
--- /dev/null
+++ b/src/hotspot/share/gc/x/xUncommitter.cpp
@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/xHeap.inline.hpp"
+#include "gc/x/xLock.inline.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xUncommitter.hpp"
+#include "jfr/jfrEvents.hpp"
+#include "logging/log.hpp"
+
+static const XStatCounter XCounterUncommit("Memory", "Uncommit", XStatUnitBytesPerSecond);
+
+XUncommitter::XUncommitter(XPageAllocator* page_allocator) :
+    _page_allocator(page_allocator),
+    _lock(),
+    _stop(false) {
+  set_name("XUncommitter");
+  create_and_start();
+}
+
+bool XUncommitter::wait(uint64_t timeout) const {
+  XLocker<XConditionLock> locker(&_lock);
+  while (!ZUncommit && !_stop) {
+    _lock.wait();
+  }
+
+  if (!_stop && timeout > 0) {
+    log_debug(gc, heap)("Uncommit Timeout: " UINT64_FORMAT "s", timeout);
+    _lock.wait(timeout * MILLIUNITS);
+  }
+
+  return !_stop;
+}
+
+bool XUncommitter::should_continue() const {
+  XLocker<XConditionLock> locker(&_lock);
+  return !_stop;
+}
+
+void XUncommitter::run_service() {
+  uint64_t timeout = 0;
+
+  while (wait(timeout)) {
+    EventZUncommit event;
+    size_t uncommitted = 0;
+
+    while (should_continue()) {
+      // Uncommit chunk
+      const size_t flushed = _page_allocator->uncommit(&timeout);
+      if (flushed == 0) {
+        // Done
+        break;
+      }
+
+      uncommitted += flushed;
+    }
+
+    if (uncommitted > 0) {
+      // Update statistics
+      XStatInc(XCounterUncommit, uncommitted);
+      log_info(gc, heap)("Uncommitted: " SIZE_FORMAT "M(%.0f%%)",
+                         uncommitted / M, percent_of(uncommitted, XHeap::heap()->max_capacity()));
+
+      // Send event
+      event.commit(uncommitted);
+    }
+  }
+}
+
+void XUncommitter::stop_service() {
+  XLocker<XConditionLock> locker(&_lock);
+  _stop = true;
+  _lock.notify_all();
+}
diff --git a/src/hotspot/share/gc/x/xUncommitter.hpp b/src/hotspot/share/gc/x/xUncommitter.hpp
new file mode 100644
index 00000000000..9f6212fa98d
--- /dev/null
+++ b/src/hotspot/share/gc/x/xUncommitter.hpp
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XUNCOMMITTER_HPP
+#define SHARE_GC_X_XUNCOMMITTER_HPP
+
+#include "gc/shared/concurrentGCThread.hpp"
+#include "gc/x/xLock.hpp"
+
+class XPageAllocation;
+
+class XUncommitter : public ConcurrentGCThread {
+private:
+  XPageAllocator* const  _page_allocator;
+  mutable XConditionLock _lock;
+  bool                   _stop;
+
+  bool wait(uint64_t timeout) const;
+  bool should_continue() const;
+
+protected:
+  virtual void run_service();
+  virtual void stop_service();
+
+public:
+  XUncommitter(XPageAllocator* page_allocator);
+};
+
+#endif // SHARE_GC_X_XUNCOMMITTER_HPP
diff --git a/src/hotspot/share/gc/x/xUnload.cpp b/src/hotspot/share/gc/x/xUnload.cpp
new file mode 100644
index 00000000000..66bdf2e222b
--- /dev/null
+++ b/src/hotspot/share/gc/x/xUnload.cpp
@@ -0,0 +1,176 @@
+/*
+ * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "classfile/classLoaderDataGraph.hpp"
+#include "classfile/systemDictionary.hpp"
+#include "code/codeBehaviours.hpp"
+#include "code/codeCache.hpp"
+#include "code/dependencyContext.hpp"
+#include "gc/shared/gcBehaviours.hpp"
+#include "gc/shared/suspendibleThreadSet.hpp"
+#include "gc/x/xBarrier.inline.hpp"
+#include "gc/x/xLock.inline.hpp"
+#include "gc/x/xNMethod.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xUnload.hpp"
+#include "memory/metaspaceUtils.hpp"
+#include "oops/access.inline.hpp"
+
+static const XStatSubPhase XSubPhaseConcurrentClassesUnlink("Concurrent Classes Unlink");
+static const XStatSubPhase XSubPhaseConcurrentClassesPurge("Concurrent Classes Purge");
+
+class XPhantomIsAliveObjectClosure : public BoolObjectClosure {
+public:
+  virtual bool do_object_b(oop o) {
+    return XBarrier::is_alive_barrier_on_phantom_oop(o);
+  }
+};
+
+class XIsUnloadingOopClosure : public OopClosure {
+private:
+  XPhantomIsAliveObjectClosure _is_alive;
+  bool                         _is_unloading;
+
+public:
+  XIsUnloadingOopClosure() :
+      _is_alive(),
+      _is_unloading(false) {}
+
+  virtual void do_oop(oop* p) {
+    const oop o = RawAccess<>::oop_load(p);
+    if (o != NULL && !_is_alive.do_object_b(o)) {
+      _is_unloading = true;
+    }
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+
+  bool is_unloading() const {
+    return _is_unloading;
+  }
+};
+
+class XIsUnloadingBehaviour : public IsUnloadingBehaviour {
+public:
+  virtual bool has_dead_oop(CompiledMethod* method) const {
+    nmethod* const nm = method->as_nmethod();
+    XReentrantLock* const lock = XNMethod::lock_for_nmethod(nm);
+    XLocker<XReentrantLock> locker(lock);
+    XIsUnloadingOopClosure cl;
+    XNMethod::nmethod_oops_do_inner(nm, &cl);
+    return cl.is_unloading();
+  }
+};
+
+class XCompiledICProtectionBehaviour : public CompiledICProtectionBehaviour {
+public:
+  virtual bool lock(CompiledMethod* method) {
+    nmethod* const nm = method->as_nmethod();
+    XReentrantLock* const lock = XNMethod::lock_for_nmethod(nm);
+    lock->lock();
+    return true;
+  }
+
+  virtual void unlock(CompiledMethod* method) {
+    nmethod* const nm = method->as_nmethod();
+    XReentrantLock* const lock = XNMethod::lock_for_nmethod(nm);
+    lock->unlock();
+  }
+
+  virtual bool is_safe(CompiledMethod* method) {
+    if (SafepointSynchronize::is_at_safepoint()) {
+      return true;
+    }
+
+    nmethod* const nm = method->as_nmethod();
+    XReentrantLock* const lock = XNMethod::lock_for_nmethod(nm);
+    return lock->is_owned();
+  }
+};
+
+XUnload::XUnload(XWorkers* workers) :
+    _workers(workers) {
+
+  if (!ClassUnloading) {
+    return;
+  }
+
+  static XIsUnloadingBehaviour is_unloading_behaviour;
+  IsUnloadingBehaviour::set_current(&is_unloading_behaviour);
+
+  static XCompiledICProtectionBehaviour ic_protection_behaviour;
+  CompiledICProtectionBehaviour::set_current(&ic_protection_behaviour);
+}
+
+void XUnload::prepare() {
+  if (!ClassUnloading) {
+    return;
+  }
+
+  CodeCache::increment_unloading_cycle();
+  DependencyContext::cleaning_start();
+}
+
+void XUnload::unlink() {
+  if (!ClassUnloading) {
+    return;
+  }
+
+  XStatTimer timer(XSubPhaseConcurrentClassesUnlink);
+  SuspendibleThreadSetJoiner sts;
+  bool unloading_occurred;
+
+  {
+    MutexLocker ml(ClassLoaderDataGraph_lock);
+    unloading_occurred = SystemDictionary::do_unloading(XStatPhase::timer());
+  }
+
+  Klass::clean_weak_klass_links(unloading_occurred);
+  XNMethod::unlink(_workers, unloading_occurred);
+  DependencyContext::cleaning_end();
+}
+
+void XUnload::purge() {
+  if (!ClassUnloading) {
+    return;
+  }
+
+  XStatTimer timer(XSubPhaseConcurrentClassesPurge);
+
+  {
+    SuspendibleThreadSetJoiner sts;
+    XNMethod::purge();
+  }
+
+  ClassLoaderDataGraph::purge(/*at_safepoint*/false);
+  CodeCache::purge_exception_caches();
+}
+
+void XUnload::finish() {
+  // Resize and verify metaspace
+  MetaspaceGC::compute_new_size();
+  DEBUG_ONLY(MetaspaceUtils::verify();)
+}
diff --git a/src/hotspot/share/gc/x/xUnload.hpp b/src/hotspot/share/gc/x/xUnload.hpp
new file mode 100644
index 00000000000..df6ba7ed2eb
--- /dev/null
+++ b/src/hotspot/share/gc/x/xUnload.hpp
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XUNLOAD_HPP
+#define SHARE_GC_X_XUNLOAD_HPP
+
+class XWorkers;
+
+class XUnload {
+private:
+  XWorkers* const _workers;
+
+public:
+  XUnload(XWorkers* workers);
+
+  void prepare();
+  void unlink();
+  void purge();
+  void finish();
+};
+
+#endif // SHARE_GC_X_XUNLOAD_HPP
diff --git a/src/hotspot/share/gc/x/xUnmapper.cpp b/src/hotspot/share/gc/x/xUnmapper.cpp
new file mode 100644
index 00000000000..baa09769074
--- /dev/null
+++ b/src/hotspot/share/gc/x/xUnmapper.cpp
@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/xList.inline.hpp"
+#include "gc/x/xLock.inline.hpp"
+#include "gc/x/xPage.inline.hpp"
+#include "gc/x/xPageAllocator.hpp"
+#include "gc/x/xUnmapper.hpp"
+#include "jfr/jfrEvents.hpp"
+#include "runtime/globals.hpp"
+
+XUnmapper::XUnmapper(XPageAllocator* page_allocator) :
+    _page_allocator(page_allocator),
+    _lock(),
+    _queue(),
+    _stop(false) {
+  set_name("XUnmapper");
+  create_and_start();
+}
+
+XPage* XUnmapper::dequeue() {
+  XLocker<XConditionLock> locker(&_lock);
+
+  for (;;) {
+    if (_stop) {
+      return NULL;
+    }
+
+    XPage* const page = _queue.remove_first();
+    if (page != NULL) {
+      return page;
+    }
+
+    _lock.wait();
+  }
+}
+
+void XUnmapper::do_unmap_and_destroy_page(XPage* page) const {
+  EventZUnmap event;
+  const size_t unmapped = page->size();
+
+  // Unmap and destroy
+  _page_allocator->unmap_page(page);
+  _page_allocator->destroy_page(page);
+
+  // Send event
+  event.commit(unmapped);
+}
+
+void XUnmapper::unmap_and_destroy_page(XPage* page) {
+  // Asynchronous unmap and destroy is not supported with ZVerifyViews
+  if (ZVerifyViews) {
+    // Immediately unmap and destroy
+    do_unmap_and_destroy_page(page);
+  } else {
+    // Enqueue for asynchronous unmap and destroy
+    XLocker<XConditionLock> locker(&_lock);
+    _queue.insert_last(page);
+    _lock.notify_all();
+  }
+}
+
+void XUnmapper::run_service() {
+  for (;;) {
+    XPage* const page = dequeue();
+    if (page == NULL) {
+      // Stop
+      return;
+    }
+
+    do_unmap_and_destroy_page(page);
+  }
+}
+
+void XUnmapper::stop_service() {
+  XLocker<XConditionLock> locker(&_lock);
+  _stop = true;
+  _lock.notify_all();
+}
diff --git a/src/hotspot/share/gc/x/xUnmapper.hpp b/src/hotspot/share/gc/x/xUnmapper.hpp
new file mode 100644
index 00000000000..9a2651ea6eb
--- /dev/null
+++ b/src/hotspot/share/gc/x/xUnmapper.hpp
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XUNMAPPER_HPP
+#define SHARE_GC_X_XUNMAPPER_HPP
+
+#include "gc/shared/concurrentGCThread.hpp"
+#include "gc/x/xList.hpp"
+#include "gc/x/xLock.hpp"
+
+class XPage;
+class XPageAllocator;
+
+class XUnmapper : public ConcurrentGCThread {
+private:
+  XPageAllocator* const _page_allocator;
+  XConditionLock        _lock;
+  XList<XPage>          _queue;
+  bool                  _stop;
+
+  XPage* dequeue();
+  void do_unmap_and_destroy_page(XPage* page) const;
+
+protected:
+  virtual void run_service();
+  virtual void stop_service();
+
+public:
+  XUnmapper(XPageAllocator* page_allocator);
+
+  void unmap_and_destroy_page(XPage* page);
+};
+
+#endif // SHARE_GC_X_XUNMAPPER_HPP
diff --git a/src/hotspot/share/gc/x/xUtils.hpp b/src/hotspot/share/gc/x/xUtils.hpp
new file mode 100644
index 00000000000..26f14c0e98f
--- /dev/null
+++ b/src/hotspot/share/gc/x/xUtils.hpp
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XUTILS_HPP
+#define SHARE_GC_X_XUTILS_HPP
+
+#include "memory/allStatic.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class XUtils : public AllStatic {
+public:
+  // Allocation
+  static uintptr_t alloc_aligned(size_t alignment, size_t size);
+
+  // Size conversion
+  static size_t bytes_to_words(size_t size_in_words);
+  static size_t words_to_bytes(size_t size_in_words);
+
+  // Object
+  static size_t object_size(uintptr_t addr);
+  static void object_copy_disjoint(uintptr_t from, uintptr_t to, size_t size);
+  static void object_copy_conjoint(uintptr_t from, uintptr_t to, size_t size);
+};
+
+#endif // SHARE_GC_X_XUTILS_HPP
diff --git a/src/hotspot/share/gc/x/xUtils.inline.hpp b/src/hotspot/share/gc/x/xUtils.inline.hpp
new file mode 100644
index 00000000000..09180959311
--- /dev/null
+++ b/src/hotspot/share/gc/x/xUtils.inline.hpp
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XUTILS_INLINE_HPP
+#define SHARE_GC_X_XUTILS_INLINE_HPP
+
+#include "gc/x/xUtils.hpp"
+
+#include "gc/x/xOop.inline.hpp"
+#include "oops/oop.inline.hpp"
+#include "utilities/align.hpp"
+#include "utilities/copy.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+inline size_t XUtils::bytes_to_words(size_t size_in_bytes) {
+  assert(is_aligned(size_in_bytes, BytesPerWord), "Size not word aligned");
+  return size_in_bytes >> LogBytesPerWord;
+}
+
+inline size_t XUtils::words_to_bytes(size_t size_in_words) {
+  return size_in_words << LogBytesPerWord;
+}
+
+inline size_t XUtils::object_size(uintptr_t addr) {
+  return words_to_bytes(XOop::from_address(addr)->size());
+}
+
+inline void XUtils::object_copy_disjoint(uintptr_t from, uintptr_t to, size_t size) {
+  Copy::aligned_disjoint_words((HeapWord*)from, (HeapWord*)to, bytes_to_words(size));
+}
+
+inline void XUtils::object_copy_conjoint(uintptr_t from, uintptr_t to, size_t size) {
+  if (from != to) {
+    Copy::aligned_conjoint_words((HeapWord*)from, (HeapWord*)to, bytes_to_words(size));
+  }
+}
+
+#endif // SHARE_GC_X_XUTILS_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xValue.hpp b/src/hotspot/share/gc/x/xValue.hpp
new file mode 100644
index 00000000000..4b2838c8a2c
--- /dev/null
+++ b/src/hotspot/share/gc/x/xValue.hpp
@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XVALUE_HPP
+#define SHARE_GC_X_XVALUE_HPP
+
+#include "memory/allStatic.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+//
+// Storage
+//
+
+template <typename S>
+class XValueStorage : public AllStatic {
+private:
+  static uintptr_t _top;
+  static uintptr_t _end;
+
+public:
+  static const size_t offset = 4 * K;
+
+  static uintptr_t alloc(size_t size);
+};
+
+class XContendedStorage : public XValueStorage<XContendedStorage> {
+public:
+  static size_t alignment();
+  static uint32_t count();
+  static uint32_t id();
+};
+
+class XPerCPUStorage : public XValueStorage<XPerCPUStorage> {
+public:
+  static size_t alignment();
+  static uint32_t count();
+  static uint32_t id();
+};
+
+class XPerNUMAStorage : public XValueStorage<XPerNUMAStorage> {
+public:
+  static size_t alignment();
+  static uint32_t count();
+  static uint32_t id();
+};
+
+class XPerWorkerStorage : public XValueStorage<XPerWorkerStorage> {
+public:
+  static size_t alignment();
+  static uint32_t count();
+  static uint32_t id();
+};
+
+//
+// Value
+//
+
+template <typename S, typename T>
+class XValue : public CHeapObj<mtGC> {
+private:
+  const uintptr_t _addr;
+
+  uintptr_t value_addr(uint32_t value_id) const;
+
+public:
+  XValue();
+  XValue(const T& value);
+
+  const T* addr(uint32_t value_id = S::id()) const;
+  T* addr(uint32_t value_id = S::id());
+
+  const T& get(uint32_t value_id = S::id()) const;
+  T& get(uint32_t value_id = S::id());
+
+  void set(const T& value, uint32_t value_id = S::id());
+  void set_all(const T& value);
+};
+
+template <typename T> using XContended = XValue<XContendedStorage, T>;
+template <typename T> using XPerCPU = XValue<XPerCPUStorage, T>;
+template <typename T> using XPerNUMA = XValue<XPerNUMAStorage, T>;
+template <typename T> using XPerWorker = XValue<XPerWorkerStorage, T>;
+
+//
+// Iterator
+//
+
+template <typename S, typename T>
+class XValueIterator {
+private:
+  XValue<S, T>* const _value;
+  uint32_t            _value_id;
+
+public:
+  XValueIterator(XValue<S, T>* value);
+
+  bool next(T** value);
+};
+
+template <typename T> using XPerCPUIterator = XValueIterator<XPerCPUStorage, T>;
+template <typename T> using XPerNUMAIterator = XValueIterator<XPerNUMAStorage, T>;
+template <typename T> using XPerWorkerIterator = XValueIterator<XPerWorkerStorage, T>;
+
+template <typename S, typename T>
+class XValueConstIterator {
+private:
+  const XValue<S, T>* const _value;
+  uint32_t                  _value_id;
+
+public:
+  XValueConstIterator(const XValue<S, T>* value);
+
+  bool next(const T** value);
+};
+
+template <typename T> using XPerCPUConstIterator = XValueConstIterator<XPerCPUStorage, T>;
+template <typename T> using XPerNUMAConstIterator = XValueConstIterator<XPerNUMAStorage, T>;
+template <typename T> using XPerWorkerConstIterator = XValueConstIterator<XPerWorkerStorage, T>;
+
+#endif // SHARE_GC_X_XVALUE_HPP
diff --git a/src/hotspot/share/gc/x/xValue.inline.hpp b/src/hotspot/share/gc/x/xValue.inline.hpp
new file mode 100644
index 00000000000..1b12eb7d555
--- /dev/null
+++ b/src/hotspot/share/gc/x/xValue.inline.hpp
@@ -0,0 +1,210 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XVALUE_INLINE_HPP
+#define SHARE_GC_X_XVALUE_INLINE_HPP
+
+#include "gc/x/xValue.hpp"
+
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/xCPU.inline.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xNUMA.hpp"
+#include "gc/x/xThread.inline.hpp"
+#include "gc/x/xUtils.hpp"
+#include "runtime/globals.hpp"
+#include "utilities/align.hpp"
+
+//
+// Storage
+//
+
+template <typename T> uintptr_t XValueStorage<T>::_end = 0;
+template <typename T> uintptr_t XValueStorage<T>::_top = 0;
+
+template <typename S>
+uintptr_t XValueStorage<S>::alloc(size_t size) {
+  assert(size <= offset, "Allocation too large");
+
+  // Allocate entry in existing memory block
+  const uintptr_t addr = align_up(_top, S::alignment());
+  _top = addr + size;
+
+  if (_top < _end) {
+    // Success
+    return addr;
+  }
+
+  // Allocate new block of memory
+  const size_t block_alignment = offset;
+  const size_t block_size = offset * S::count();
+  _top = XUtils::alloc_aligned(block_alignment, block_size);
+  _end = _top + offset;
+
+  // Retry allocation
+  return alloc(size);
+}
+
+inline size_t XContendedStorage::alignment() {
+  return XCacheLineSize;
+}
+
+inline uint32_t XContendedStorage::count() {
+  return 1;
+}
+
+inline uint32_t XContendedStorage::id() {
+  return 0;
+}
+
+inline size_t XPerCPUStorage::alignment() {
+  return sizeof(uintptr_t);
+}
+
+inline uint32_t XPerCPUStorage::count() {
+  return XCPU::count();
+}
+
+inline uint32_t XPerCPUStorage::id() {
+  return XCPU::id();
+}
+
+inline size_t XPerNUMAStorage::alignment() {
+  return sizeof(uintptr_t);
+}
+
+inline uint32_t XPerNUMAStorage::count() {
+  return XNUMA::count();
+}
+
+inline uint32_t XPerNUMAStorage::id() {
+  return XNUMA::id();
+}
+
+inline size_t XPerWorkerStorage::alignment() {
+  return sizeof(uintptr_t);
+}
+
+inline uint32_t XPerWorkerStorage::count() {
+  return UseDynamicNumberOfGCThreads ? ConcGCThreads : MAX2(ConcGCThreads, ParallelGCThreads);
+}
+
+inline uint32_t XPerWorkerStorage::id() {
+  return XThread::worker_id();
+}
+
+//
+// Value
+//
+
+template <typename S, typename T>
+inline uintptr_t XValue<S, T>::value_addr(uint32_t value_id) const {
+  return _addr + (value_id * S::offset);
+}
+
+template <typename S, typename T>
+inline XValue<S, T>::XValue() :
+    _addr(S::alloc(sizeof(T))) {
+  // Initialize all instances
+  XValueIterator<S, T> iter(this);
+  for (T* addr; iter.next(&addr);) {
+    ::new (addr) T;
+  }
+}
+
+template <typename S, typename T>
+inline XValue<S, T>::XValue(const T& value) :
+    _addr(S::alloc(sizeof(T))) {
+  // Initialize all instances
+  XValueIterator<S, T> iter(this);
+  for (T* addr; iter.next(&addr);) {
+    ::new (addr) T(value);
+  }
+}
+
+template <typename S, typename T>
+inline const T* XValue<S, T>::addr(uint32_t value_id) const {
+  return reinterpret_cast<const T*>(value_addr(value_id));
+}
+
+template <typename S, typename T>
+inline T* XValue<S, T>::addr(uint32_t value_id) {
+  return reinterpret_cast<T*>(value_addr(value_id));
+}
+
+template <typename S, typename T>
+inline const T& XValue<S, T>::get(uint32_t value_id) const {
+  return *addr(value_id);
+}
+
+template <typename S, typename T>
+inline T& XValue<S, T>::get(uint32_t value_id) {
+  return *addr(value_id);
+}
+
+template <typename S, typename T>
+inline void XValue<S, T>::set(const T& value, uint32_t value_id) {
+  get(value_id) = value;
+}
+
+template <typename S, typename T>
+inline void XValue<S, T>::set_all(const T& value) {
+  XValueIterator<S, T> iter(this);
+  for (T* addr; iter.next(&addr);) {
+    *addr = value;
+  }
+}
+
+//
+// Iterator
+//
+
+template <typename S, typename T>
+inline XValueIterator<S, T>::XValueIterator(XValue<S, T>* value) :
+    _value(value),
+    _value_id(0) {}
+
+template <typename S, typename T>
+inline bool XValueIterator<S, T>::next(T** value) {
+  if (_value_id < S::count()) {
+    *value = _value->addr(_value_id++);
+    return true;
+  }
+  return false;
+}
+
+template <typename S, typename T>
+inline XValueConstIterator<S, T>::XValueConstIterator(const XValue<S, T>* value) :
+    _value(value),
+    _value_id(0) {}
+
+template <typename S, typename T>
+inline bool XValueConstIterator<S, T>::next(const T** value) {
+  if (_value_id < S::count()) {
+    *value = _value->addr(_value_id++);
+    return true;
+  }
+  return false;
+}
+
+#endif // SHARE_GC_X_XVALUE_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xVerify.cpp b/src/hotspot/share/gc/x/xVerify.cpp
new file mode 100644
index 00000000000..ed3e224091c
--- /dev/null
+++ b/src/hotspot/share/gc/x/xVerify.cpp
@@ -0,0 +1,415 @@
+/*
+ * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "classfile/classLoaderData.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xHeap.inline.hpp"
+#include "gc/x/xNMethod.hpp"
+#include "gc/x/xOop.hpp"
+#include "gc/x/xPageAllocator.hpp"
+#include "gc/x/xResurrection.hpp"
+#include "gc/x/xRootsIterator.hpp"
+#include "gc/x/xStackWatermark.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xVerify.hpp"
+#include "memory/iterator.inline.hpp"
+#include "memory/resourceArea.hpp"
+#include "oops/oop.hpp"
+#include "runtime/frame.inline.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/handles.hpp"
+#include "runtime/javaThread.hpp"
+#include "runtime/safepoint.hpp"
+#include "runtime/stackFrameStream.inline.hpp"
+#include "runtime/stackWatermark.inline.hpp"
+#include "runtime/stackWatermarkSet.inline.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/globalDefinitions.hpp"
+#include "utilities/preserveException.hpp"
+
+#define BAD_OOP_ARG(o, p)   "Bad oop " PTR_FORMAT " found at " PTR_FORMAT, p2i(o), p2i(p)
+
+static void z_verify_oop(oop* p) {
+  const oop o = RawAccess<>::oop_load(p);
+  if (o != NULL) {
+    const uintptr_t addr = XOop::to_address(o);
+    guarantee(XAddress::is_good(addr), BAD_OOP_ARG(o, p));
+    guarantee(oopDesc::is_oop(XOop::from_address(addr)), BAD_OOP_ARG(o, p));
+  }
+}
+
+static void z_verify_possibly_weak_oop(oop* p) {
+  const oop o = RawAccess<>::oop_load(p);
+  if (o != NULL) {
+    const uintptr_t addr = XOop::to_address(o);
+    guarantee(XAddress::is_good(addr) || XAddress::is_finalizable_good(addr), BAD_OOP_ARG(o, p));
+    guarantee(oopDesc::is_oop(XOop::from_address(XAddress::good(addr))), BAD_OOP_ARG(o, p));
+  }
+}
+
+class XVerifyRootClosure : public OopClosure {
+private:
+  const bool _verify_fixed;
+
+public:
+  XVerifyRootClosure(bool verify_fixed) :
+      _verify_fixed(verify_fixed) {}
+
+  virtual void do_oop(oop* p) {
+    if (_verify_fixed) {
+      z_verify_oop(p);
+    } else {
+      // Don't know the state of the oop.
+      oop obj = *p;
+      obj = NativeAccess<AS_NO_KEEPALIVE>::oop_load(&obj);
+      z_verify_oop(&obj);
+    }
+  }
+
+  virtual void do_oop(narrowOop*) {
+    ShouldNotReachHere();
+  }
+
+  bool verify_fixed() const {
+    return _verify_fixed;
+  }
+};
+
+class XVerifyCodeBlobClosure : public CodeBlobToOopClosure {
+public:
+  XVerifyCodeBlobClosure(XVerifyRootClosure* _cl) :
+      CodeBlobToOopClosure(_cl, false /* fix_relocations */) {}
+
+  virtual void do_code_blob(CodeBlob* cb) {
+    CodeBlobToOopClosure::do_code_blob(cb);
+  }
+};
+
+class XVerifyStack : public OopClosure {
+private:
+  XVerifyRootClosure* const _cl;
+  JavaThread*         const _jt;
+  uint64_t                  _last_good;
+  bool                      _verifying_bad_frames;
+
+public:
+  XVerifyStack(XVerifyRootClosure* cl, JavaThread* jt) :
+      _cl(cl),
+      _jt(jt),
+      _last_good(0),
+      _verifying_bad_frames(false) {
+    XStackWatermark* const stack_watermark = StackWatermarkSet::get<XStackWatermark>(jt, StackWatermarkKind::gc);
+
+    if (_cl->verify_fixed()) {
+      assert(stack_watermark->processing_started(), "Should already have been fixed");
+      assert(stack_watermark->processing_completed(), "Should already have been fixed");
+    } else {
+      // We don't really know the state of the stack, verify watermark.
+      if (!stack_watermark->processing_started()) {
+        _verifying_bad_frames = true;
+      } else {
+        // Not time yet to verify bad frames
+        _last_good = stack_watermark->last_processed();
+      }
+    }
+  }
+
+  void do_oop(oop* p) {
+    if (_verifying_bad_frames) {
+      const oop obj = *p;
+      guarantee(!XAddress::is_good(XOop::to_address(obj)), BAD_OOP_ARG(obj, p));
+    }
+    _cl->do_oop(p);
+  }
+
+  void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+
+  void prepare_next_frame(frame& frame) {
+    if (_cl->verify_fixed()) {
+      // All frames need to be good
+      return;
+    }
+
+    // The verification has two modes, depending on whether we have reached the
+    // last processed frame or not. Before it is reached, we expect everything to
+    // be good. After reaching it, we expect everything to be bad.
+    const uintptr_t sp = reinterpret_cast<uintptr_t>(frame.sp());
+
+    if (!_verifying_bad_frames && sp == _last_good) {
+      // Found the last good frame, now verify the bad ones
+      _verifying_bad_frames = true;
+    }
+  }
+
+  void verify_frames() {
+    XVerifyCodeBlobClosure cb_cl(_cl);
+    for (StackFrameStream frames(_jt, true /* update */, false /* process_frames */);
+         !frames.is_done();
+         frames.next()) {
+      frame& frame = *frames.current();
+      frame.oops_do(this, &cb_cl, frames.register_map(), DerivedPointerIterationMode::_ignore);
+      prepare_next_frame(frame);
+    }
+  }
+};
+
+class XVerifyOopClosure : public ClaimMetadataVisitingOopIterateClosure {
+private:
+  const bool _verify_weaks;
+
+public:
+  XVerifyOopClosure(bool verify_weaks) :
+      ClaimMetadataVisitingOopIterateClosure(ClassLoaderData::_claim_other),
+      _verify_weaks(verify_weaks) {}
+
+  virtual void do_oop(oop* p) {
+    if (_verify_weaks) {
+      z_verify_possibly_weak_oop(p);
+    } else {
+      // We should never encounter finalizable oops through strong
+      // paths. This assumes we have only visited strong roots.
+      z_verify_oop(p);
+    }
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+
+  virtual ReferenceIterationMode reference_iteration_mode() {
+    return _verify_weaks ? DO_FIELDS : DO_FIELDS_EXCEPT_REFERENT;
+  }
+
+  // Don't follow this metadata when verifying oops
+  virtual void do_method(Method* m) {}
+  virtual void do_nmethod(nmethod* nm) {}
+};
+
+typedef ClaimingCLDToOopClosure<ClassLoaderData::_claim_none> XVerifyCLDClosure;
+
+class XVerifyThreadClosure : public ThreadClosure {
+private:
+  XVerifyRootClosure* const _cl;
+
+public:
+  XVerifyThreadClosure(XVerifyRootClosure* cl) :
+      _cl(cl) {}
+
+  virtual void do_thread(Thread* thread) {
+    thread->oops_do_no_frames(_cl, NULL);
+
+    JavaThread* const jt = JavaThread::cast(thread);
+    if (!jt->has_last_Java_frame()) {
+      return;
+    }
+
+    XVerifyStack verify_stack(_cl, jt);
+    verify_stack.verify_frames();
+  }
+};
+
+class XVerifyNMethodClosure : public NMethodClosure {
+private:
+  OopClosure* const        _cl;
+  BarrierSetNMethod* const _bs_nm;
+  const bool               _verify_fixed;
+
+  bool trust_nmethod_state() const {
+    // The root iterator will visit non-processed
+    // nmethods class unloading is turned off.
+    return ClassUnloading || _verify_fixed;
+  }
+
+public:
+  XVerifyNMethodClosure(OopClosure* cl, bool verify_fixed) :
+      _cl(cl),
+      _bs_nm(BarrierSet::barrier_set()->barrier_set_nmethod()),
+      _verify_fixed(verify_fixed) {}
+
+  virtual void do_nmethod(nmethod* nm) {
+    assert(!trust_nmethod_state() || !_bs_nm->is_armed(nm), "Should not encounter any armed nmethods");
+
+    XNMethod::nmethod_oops_do(nm, _cl);
+  }
+};
+
+void XVerify::roots_strong(bool verify_fixed) {
+  assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
+  assert(!XResurrection::is_blocked(), "Invalid phase");
+
+  XVerifyRootClosure cl(verify_fixed);
+  XVerifyCLDClosure cld_cl(&cl);
+  XVerifyThreadClosure thread_cl(&cl);
+  XVerifyNMethodClosure nm_cl(&cl, verify_fixed);
+
+  XRootsIterator iter(ClassLoaderData::_claim_none);
+  iter.apply(&cl,
+             &cld_cl,
+             &thread_cl,
+             &nm_cl);
+}
+
+void XVerify::roots_weak() {
+  assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
+  assert(!XResurrection::is_blocked(), "Invalid phase");
+
+  XVerifyRootClosure cl(true /* verify_fixed */);
+  XWeakRootsIterator iter;
+  iter.apply(&cl);
+}
+
+void XVerify::objects(bool verify_weaks) {
+  assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
+  assert(XGlobalPhase == XPhaseMarkCompleted, "Invalid phase");
+  assert(!XResurrection::is_blocked(), "Invalid phase");
+
+  XVerifyOopClosure cl(verify_weaks);
+  ObjectToOopClosure object_cl(&cl);
+  XHeap::heap()->object_iterate(&object_cl, verify_weaks);
+}
+
+void XVerify::before_zoperation() {
+  // Verify strong roots
+  XStatTimerDisable disable;
+  if (ZVerifyRoots) {
+    roots_strong(false /* verify_fixed */);
+  }
+}
+
+void XVerify::after_mark() {
+  // Verify all strong roots and strong references
+  XStatTimerDisable disable;
+  if (ZVerifyRoots) {
+    roots_strong(true /* verify_fixed */);
+  }
+  if (ZVerifyObjects) {
+    objects(false /* verify_weaks */);
+  }
+}
+
+void XVerify::after_weak_processing() {
+  // Verify all roots and all references
+  XStatTimerDisable disable;
+  if (ZVerifyRoots) {
+    roots_strong(true /* verify_fixed */);
+    roots_weak();
+  }
+  if (ZVerifyObjects) {
+    objects(true /* verify_weaks */);
+  }
+}
+
+template <bool Map>
+class XPageDebugMapOrUnmapClosure : public XPageClosure {
+private:
+  const XPageAllocator* const _allocator;
+
+public:
+  XPageDebugMapOrUnmapClosure(const XPageAllocator* allocator) :
+      _allocator(allocator) {}
+
+  void do_page(const XPage* page) {
+    if (Map) {
+      _allocator->debug_map_page(page);
+    } else {
+      _allocator->debug_unmap_page(page);
+    }
+  }
+};
+
+XVerifyViewsFlip::XVerifyViewsFlip(const XPageAllocator* allocator) :
+    _allocator(allocator) {
+  if (ZVerifyViews) {
+    // Unmap all pages
+    XPageDebugMapOrUnmapClosure<false /* Map */> cl(_allocator);
+    XHeap::heap()->pages_do(&cl);
+  }
+}
+
+XVerifyViewsFlip::~XVerifyViewsFlip() {
+  if (ZVerifyViews) {
+    // Map all pages
+    XPageDebugMapOrUnmapClosure<true /* Map */> cl(_allocator);
+    XHeap::heap()->pages_do(&cl);
+  }
+}
+
+#ifdef ASSERT
+
+class XVerifyBadOopClosure : public OopClosure {
+public:
+  virtual void do_oop(oop* p) {
+    const oop o = *p;
+    assert(!XAddress::is_good(XOop::to_address(o)), "Should not be good: " PTR_FORMAT, p2i(o));
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+};
+
+// This class encapsulates various marks we need to deal with calling the
+// frame iteration code from arbitrary points in the runtime. It is mostly
+// due to problems that we might want to eventually clean up inside of the
+// frame iteration code, such as creating random handles even though there
+// is no safepoint to protect against, and fiddling around with exceptions.
+class StackWatermarkProcessingMark {
+  ResetNoHandleMark     _rnhm;
+  HandleMark            _hm;
+  PreserveExceptionMark _pem;
+  ResourceMark          _rm;
+
+public:
+  StackWatermarkProcessingMark(Thread* thread) :
+      _rnhm(),
+      _hm(thread),
+      _pem(thread),
+      _rm(thread) {}
+};
+
+void XVerify::verify_frame_bad(const frame& fr, RegisterMap& register_map) {
+  XVerifyBadOopClosure verify_cl;
+  fr.oops_do(&verify_cl, NULL, &register_map, DerivedPointerIterationMode::_ignore);
+}
+
+void XVerify::verify_thread_head_bad(JavaThread* jt) {
+  XVerifyBadOopClosure verify_cl;
+  jt->oops_do_no_frames(&verify_cl, NULL);
+}
+
+void XVerify::verify_thread_frames_bad(JavaThread* jt) {
+  if (jt->has_last_Java_frame()) {
+    XVerifyBadOopClosure verify_cl;
+    StackWatermarkProcessingMark swpm(Thread::current());
+    // Traverse the execution stack
+    for (StackFrameStream fst(jt, true /* update */, false /* process_frames */); !fst.is_done(); fst.next()) {
+      fst.current()->oops_do(&verify_cl, NULL /* code_cl */, fst.register_map(), DerivedPointerIterationMode::_ignore);
+    }
+  }
+}
+
+#endif // ASSERT
diff --git a/src/hotspot/share/gc/x/xVerify.hpp b/src/hotspot/share/gc/x/xVerify.hpp
new file mode 100644
index 00000000000..bbe10f376fa
--- /dev/null
+++ b/src/hotspot/share/gc/x/xVerify.hpp
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XVERIFY_HPP
+#define SHARE_GC_X_XVERIFY_HPP
+
+#include "memory/allStatic.hpp"
+
+class frame;
+class XPageAllocator;
+
+class XVerify : public AllStatic {
+private:
+  static void roots_strong(bool verify_fixed);
+  static void roots_weak();
+
+  static void objects(bool verify_weaks);
+
+public:
+  static void before_zoperation();
+  static void after_mark();
+  static void after_weak_processing();
+
+  static void verify_thread_head_bad(JavaThread* thread) NOT_DEBUG_RETURN;
+  static void verify_thread_frames_bad(JavaThread* thread) NOT_DEBUG_RETURN;
+  static void verify_frame_bad(const frame& fr, RegisterMap& register_map) NOT_DEBUG_RETURN;
+};
+
+class XVerifyViewsFlip {
+private:
+  const XPageAllocator* const _allocator;
+
+public:
+  XVerifyViewsFlip(const XPageAllocator* allocator);
+  ~XVerifyViewsFlip();
+};
+
+#endif // SHARE_GC_X_XVERIFY_HPP
diff --git a/src/hotspot/share/gc/x/xVirtualMemory.cpp b/src/hotspot/share/gc/x/xVirtualMemory.cpp
new file mode 100644
index 00000000000..650f22f3283
--- /dev/null
+++ b/src/hotspot/share/gc/x/xVirtualMemory.cpp
@@ -0,0 +1,208 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xAddressSpaceLimit.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xVirtualMemory.inline.hpp"
+#include "services/memTracker.hpp"
+#include "utilities/align.hpp"
+#include "utilities/debug.hpp"
+
+XVirtualMemoryManager::XVirtualMemoryManager(size_t max_capacity) :
+    _manager(),
+    _reserved(0),
+    _initialized(false) {
+
+  // Check max supported heap size
+  if (max_capacity > XAddressOffsetMax) {
+    log_error_p(gc)("Java heap too large (max supported heap size is " SIZE_FORMAT "G)",
+                    XAddressOffsetMax / G);
+    return;
+  }
+
+  // Initialize platform specific parts before reserving address space
+  pd_initialize_before_reserve();
+
+  // Reserve address space
+  if (!reserve(max_capacity)) {
+    log_error_pd(gc)("Failed to reserve enough address space for Java heap");
+    return;
+  }
+
+  // Initialize platform specific parts after reserving address space
+  pd_initialize_after_reserve();
+
+  // Successfully initialized
+  _initialized = true;
+}
+
+size_t XVirtualMemoryManager::reserve_discontiguous(uintptr_t start, size_t size, size_t min_range) {
+  if (size < min_range) {
+    // Too small
+    return 0;
+  }
+
+  assert(is_aligned(size, XGranuleSize), "Misaligned");
+
+  if (reserve_contiguous(start, size)) {
+    return size;
+  }
+
+  const size_t half = size / 2;
+  if (half < min_range) {
+    // Too small
+    return 0;
+  }
+
+  // Divide and conquer
+  const size_t first_part = align_down(half, XGranuleSize);
+  const size_t second_part = size - first_part;
+  return reserve_discontiguous(start, first_part, min_range) +
+         reserve_discontiguous(start + first_part, second_part, min_range);
+}
+
+size_t XVirtualMemoryManager::reserve_discontiguous(size_t size) {
+  // Don't try to reserve address ranges smaller than 1% of the requested size.
+  // This avoids an explosion of reservation attempts in case large parts of the
+  // address space is already occupied.
+  const size_t min_range = align_up(size / 100, XGranuleSize);
+  size_t start = 0;
+  size_t reserved = 0;
+
+  // Reserve size somewhere between [0, XAddressOffsetMax)
+  while (reserved < size && start < XAddressOffsetMax) {
+    const size_t remaining = MIN2(size - reserved, XAddressOffsetMax - start);
+    reserved += reserve_discontiguous(start, remaining, min_range);
+    start += remaining;
+  }
+
+  return reserved;
+}
+
+bool XVirtualMemoryManager::reserve_contiguous(uintptr_t start, size_t size) {
+  assert(is_aligned(size, XGranuleSize), "Must be granule aligned");
+
+  // Reserve address views
+  const uintptr_t marked0 = XAddress::marked0(start);
+  const uintptr_t marked1 = XAddress::marked1(start);
+  const uintptr_t remapped = XAddress::remapped(start);
+
+  // Reserve address space
+  if (!pd_reserve(marked0, size)) {
+    return false;
+  }
+
+  if (!pd_reserve(marked1, size)) {
+    pd_unreserve(marked0, size);
+    return false;
+  }
+
+  if (!pd_reserve(remapped, size)) {
+    pd_unreserve(marked0, size);
+    pd_unreserve(marked1, size);
+    return false;
+  }
+
+  // Register address views with native memory tracker
+  nmt_reserve(marked0, size);
+  nmt_reserve(marked1, size);
+  nmt_reserve(remapped, size);
+
+  // Make the address range free
+  _manager.free(start, size);
+
+  return true;
+}
+
+bool XVirtualMemoryManager::reserve_contiguous(size_t size) {
+  // Allow at most 8192 attempts spread evenly across [0, XAddressOffsetMax)
+  const size_t unused = XAddressOffsetMax - size;
+  const size_t increment = MAX2(align_up(unused / 8192, XGranuleSize), XGranuleSize);
+
+  for (size_t start = 0; start + size <= XAddressOffsetMax; start += increment) {
+    if (reserve_contiguous(start, size)) {
+      // Success
+      return true;
+    }
+  }
+
+  // Failed
+  return false;
+}
+
+bool XVirtualMemoryManager::reserve(size_t max_capacity) {
+  const size_t limit = MIN2(XAddressOffsetMax, XAddressSpaceLimit::heap_view());
+  const size_t size = MIN2(max_capacity * XVirtualToPhysicalRatio, limit);
+
+  size_t reserved = size;
+  bool contiguous = true;
+
+  // Prefer a contiguous address space
+  if (!reserve_contiguous(size)) {
+    // Fall back to a discontiguous address space
+    reserved = reserve_discontiguous(size);
+    contiguous = false;
+  }
+
+  log_info_p(gc, init)("Address Space Type: %s/%s/%s",
+                       (contiguous ? "Contiguous" : "Discontiguous"),
+                       (limit == XAddressOffsetMax ? "Unrestricted" : "Restricted"),
+                       (reserved == size ? "Complete" : "Degraded"));
+  log_info_p(gc, init)("Address Space Size: " SIZE_FORMAT "M x " SIZE_FORMAT " = " SIZE_FORMAT "M",
+                       reserved / M, XHeapViews, (reserved * XHeapViews) / M);
+
+  // Record reserved
+  _reserved = reserved;
+
+  return reserved >= max_capacity;
+}
+
+void XVirtualMemoryManager::nmt_reserve(uintptr_t start, size_t size) {
+  MemTracker::record_virtual_memory_reserve((void*)start, size, CALLER_PC);
+  MemTracker::record_virtual_memory_type((void*)start, mtJavaHeap);
+}
+
+bool XVirtualMemoryManager::is_initialized() const {
+  return _initialized;
+}
+
+XVirtualMemory XVirtualMemoryManager::alloc(size_t size, bool force_low_address) {
+  uintptr_t start;
+
+  // Small pages are allocated at low addresses, while medium/large pages
+  // are allocated at high addresses (unless forced to be at a low address).
+  if (force_low_address || size <= XPageSizeSmall) {
+    start = _manager.alloc_low_address(size);
+  } else {
+    start = _manager.alloc_high_address(size);
+  }
+
+  return XVirtualMemory(start, size);
+}
+
+void XVirtualMemoryManager::free(const XVirtualMemory& vmem) {
+  _manager.free(vmem.start(), vmem.size());
+}
diff --git a/src/hotspot/share/gc/x/xVirtualMemory.hpp b/src/hotspot/share/gc/x/xVirtualMemory.hpp
new file mode 100644
index 00000000000..c9e5c67ea57
--- /dev/null
+++ b/src/hotspot/share/gc/x/xVirtualMemory.hpp
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XVIRTUALMEMORY_HPP
+#define SHARE_GC_X_XVIRTUALMEMORY_HPP
+
+#include "gc/x/xMemory.hpp"
+
+class VMStructs;
+
+class XVirtualMemory {
+  friend class ::VMStructs;
+
+private:
+  uintptr_t _start;
+  uintptr_t _end;
+
+public:
+  XVirtualMemory();
+  XVirtualMemory(uintptr_t start, size_t size);
+
+  bool is_null() const;
+  uintptr_t start() const;
+  uintptr_t end() const;
+  size_t size() const;
+
+  XVirtualMemory split(size_t size);
+};
+
+class XVirtualMemoryManager {
+private:
+  XMemoryManager _manager;
+  uintptr_t      _reserved;
+  bool           _initialized;
+
+  // Platform specific implementation
+  void pd_initialize_before_reserve();
+  void pd_initialize_after_reserve();
+  bool pd_reserve(uintptr_t addr, size_t size);
+  void pd_unreserve(uintptr_t addr, size_t size);
+
+  bool reserve_contiguous(uintptr_t start, size_t size);
+  bool reserve_contiguous(size_t size);
+  size_t reserve_discontiguous(uintptr_t start, size_t size, size_t min_range);
+  size_t reserve_discontiguous(size_t size);
+  bool reserve(size_t max_capacity);
+
+  void nmt_reserve(uintptr_t start, size_t size);
+
+public:
+  XVirtualMemoryManager(size_t max_capacity);
+
+  bool is_initialized() const;
+
+  size_t reserved() const;
+  uintptr_t lowest_available_address() const;
+
+  XVirtualMemory alloc(size_t size, bool force_low_address);
+  void free(const XVirtualMemory& vmem);
+};
+
+#endif // SHARE_GC_X_XVIRTUALMEMORY_HPP
diff --git a/src/hotspot/share/gc/x/xVirtualMemory.inline.hpp b/src/hotspot/share/gc/x/xVirtualMemory.inline.hpp
new file mode 100644
index 00000000000..8c834b42c7f
--- /dev/null
+++ b/src/hotspot/share/gc/x/xVirtualMemory.inline.hpp
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XVIRTUALMEMORY_INLINE_HPP
+#define SHARE_GC_X_XVIRTUALMEMORY_INLINE_HPP
+
+#include "gc/x/xVirtualMemory.hpp"
+
+#include "gc/x/xMemory.inline.hpp"
+
+inline XVirtualMemory::XVirtualMemory() :
+    _start(UINTPTR_MAX),
+    _end(UINTPTR_MAX) {}
+
+inline XVirtualMemory::XVirtualMemory(uintptr_t start, size_t size) :
+    _start(start),
+    _end(start + size) {}
+
+inline bool XVirtualMemory::is_null() const {
+  return _start == UINTPTR_MAX;
+}
+
+inline uintptr_t XVirtualMemory::start() const {
+  return _start;
+}
+
+inline uintptr_t XVirtualMemory::end() const {
+  return _end;
+}
+
+inline size_t XVirtualMemory::size() const {
+  return _end - _start;
+}
+
+inline XVirtualMemory XVirtualMemory::split(size_t size) {
+  _start += size;
+  return XVirtualMemory(_start - size, size);
+}
+
+inline size_t XVirtualMemoryManager::reserved() const {
+  return _reserved;
+}
+
+inline uintptr_t XVirtualMemoryManager::lowest_available_address() const {
+  return _manager.peek_low_address();
+}
+
+#endif // SHARE_GC_X_XVIRTUALMEMORY_INLINE_HPP
diff --git a/src/hotspot/share/gc/x/xWeakRootsProcessor.cpp b/src/hotspot/share/gc/x/xWeakRootsProcessor.cpp
new file mode 100644
index 00000000000..27eaead98fe
--- /dev/null
+++ b/src/hotspot/share/gc/x/xWeakRootsProcessor.cpp
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xBarrier.inline.hpp"
+#include "gc/x/xRootsIterator.hpp"
+#include "gc/x/xTask.hpp"
+#include "gc/x/xWeakRootsProcessor.hpp"
+#include "gc/x/xWorkers.hpp"
+
+class XPhantomCleanOopClosure : public OopClosure {
+public:
+  virtual void do_oop(oop* p) {
+    // Read the oop once, to make sure the liveness check
+    // and the later clearing uses the same value.
+    const oop obj = Atomic::load(p);
+    if (XBarrier::is_alive_barrier_on_phantom_oop(obj)) {
+      XBarrier::keep_alive_barrier_on_phantom_oop_field(p);
+    } else {
+      // The destination could have been modified/reused, in which case
+      // we don't want to clear it. However, no one could write the same
+      // oop here again (the object would be strongly live and we would
+      // not consider clearing such oops), so therefore we don't have an
+      // ABA problem here.
+      Atomic::cmpxchg(p, obj, oop(NULL));
+    }
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+};
+
+XWeakRootsProcessor::XWeakRootsProcessor(XWorkers* workers) :
+    _workers(workers) {}
+
+class XProcessWeakRootsTask : public XTask {
+private:
+  XWeakRootsIterator _weak_roots;
+
+public:
+  XProcessWeakRootsTask() :
+      XTask("XProcessWeakRootsTask"),
+      _weak_roots() {}
+
+  ~XProcessWeakRootsTask() {
+    _weak_roots.report_num_dead();
+  }
+
+  virtual void work() {
+    XPhantomCleanOopClosure cl;
+    _weak_roots.apply(&cl);
+  }
+};
+
+void XWeakRootsProcessor::process_weak_roots() {
+  XProcessWeakRootsTask task;
+  _workers->run(&task);
+}
diff --git a/src/hotspot/share/gc/x/xWeakRootsProcessor.hpp b/src/hotspot/share/gc/x/xWeakRootsProcessor.hpp
new file mode 100644
index 00000000000..c63b2702374
--- /dev/null
+++ b/src/hotspot/share/gc/x/xWeakRootsProcessor.hpp
@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XWEAKROOTSPROCESSOR_HPP
+#define SHARE_GC_X_XWEAKROOTSPROCESSOR_HPP
+
+class XWorkers;
+
+class XWeakRootsProcessor {
+private:
+  XWorkers* const _workers;
+
+public:
+  XWeakRootsProcessor(XWorkers* workers);
+
+  void process_weak_roots();
+};
+
+#endif // SHARE_GC_X_XWEAKROOTSPROCESSOR_HPP
diff --git a/src/hotspot/share/gc/x/xWorkers.cpp b/src/hotspot/share/gc/x/xWorkers.cpp
new file mode 100644
index 00000000000..642c63f0531
--- /dev/null
+++ b/src/hotspot/share/gc/x/xWorkers.cpp
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
+#include "gc/x/xLock.inline.hpp"
+#include "gc/x/xStat.hpp"
+#include "gc/x/xTask.hpp"
+#include "gc/x/xThread.hpp"
+#include "gc/x/xWorkers.hpp"
+#include "runtime/java.hpp"
+
+class XWorkersInitializeTask : public WorkerTask {
+private:
+  const uint     _nworkers;
+  uint           _started;
+  XConditionLock _lock;
+
+public:
+  XWorkersInitializeTask(uint nworkers) :
+      WorkerTask("XWorkersInitializeTask"),
+      _nworkers(nworkers),
+      _started(0),
+      _lock() {}
+
+  virtual void work(uint worker_id) {
+    // Register as worker
+    XThread::set_worker();
+
+    // Wait for all threads to start
+    XLocker<XConditionLock> locker(&_lock);
+    if (++_started == _nworkers) {
+      // All threads started
+      _lock.notify_all();
+    } else {
+      while (_started != _nworkers) {
+        _lock.wait();
+      }
+    }
+  }
+};
+
+XWorkers::XWorkers() :
+    _workers("XWorker",
+             UseDynamicNumberOfGCThreads ? ConcGCThreads : MAX2(ConcGCThreads, ParallelGCThreads)) {
+
+  if (UseDynamicNumberOfGCThreads) {
+    log_info_p(gc, init)("GC Workers: %u (dynamic)", _workers.max_workers());
+  } else {
+    log_info_p(gc, init)("GC Workers: %u/%u (static)", ConcGCThreads, _workers.max_workers());
+  }
+
+  // Initialize worker threads
+  _workers.initialize_workers();
+  _workers.set_active_workers(_workers.max_workers());
+  if (_workers.active_workers() != _workers.max_workers()) {
+    vm_exit_during_initialization("Failed to create XWorkers");
+  }
+
+  // Execute task to register threads as workers
+  XWorkersInitializeTask task(_workers.max_workers());
+  _workers.run_task(&task);
+}
+
+uint XWorkers::active_workers() const {
+  return _workers.active_workers();
+}
+
+void XWorkers::set_active_workers(uint nworkers) {
+  log_info(gc, task)("Using %u workers", nworkers);
+  _workers.set_active_workers(nworkers);
+}
+
+void XWorkers::run(XTask* task) {
+  log_debug(gc, task)("Executing Task: %s, Active Workers: %u", task->name(), active_workers());
+  XStatWorkers::at_start();
+  _workers.run_task(task->worker_task());
+  XStatWorkers::at_end();
+}
+
+void XWorkers::run_all(XTask* task) {
+  // Save number of active workers
+  const uint prev_active_workers = _workers.active_workers();
+
+  // Execute task using all workers
+  _workers.set_active_workers(_workers.max_workers());
+  log_debug(gc, task)("Executing Task: %s, Active Workers: %u", task->name(), active_workers());
+  _workers.run_task(task->worker_task());
+
+  // Restore number of active workers
+  _workers.set_active_workers(prev_active_workers);
+}
+
+void XWorkers::threads_do(ThreadClosure* tc) const {
+  _workers.threads_do(tc);
+}
diff --git a/src/hotspot/share/gc/x/xWorkers.hpp b/src/hotspot/share/gc/x/xWorkers.hpp
new file mode 100644
index 00000000000..33c49bb7fef
--- /dev/null
+++ b/src/hotspot/share/gc/x/xWorkers.hpp
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_XWORKERS_HPP
+#define SHARE_GC_X_XWORKERS_HPP
+
+#include "gc/shared/workerThread.hpp"
+
+class ThreadClosure;
+class XTask;
+
+class XWorkers {
+private:
+  WorkerThreads _workers;
+
+public:
+  XWorkers();
+
+  uint active_workers() const;
+  void set_active_workers(uint nworkers);
+
+  void run(XTask* task);
+  void run_all(XTask* task);
+
+  void threads_do(ThreadClosure* tc) const;
+};
+
+#endif // SHARE_GC_X_XWORKERS_HPP
diff --git a/src/hotspot/share/gc/x/x_globals.hpp b/src/hotspot/share/gc/x/x_globals.hpp
new file mode 100644
index 00000000000..a23c1353a68
--- /dev/null
+++ b/src/hotspot/share/gc/x/x_globals.hpp
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_X_X_GLOBALS_HPP
+#define SHARE_GC_X_X_GLOBALS_HPP
+
+#define GC_X_FLAGS(develop,                                          \
+                   develop_pd,                                       \
+                   product,                                          \
+                   product_pd,                                       \
+                   notproduct,                                       \
+                   range,                                            \
+                   constraint)                                       \
+                                                                     \
+  product(bool, ZVerifyViews, false, DIAGNOSTIC,                     \
+          "Verify heap view accesses")                               \
+                                                                     \
+// end of GC_X_FLAGS
+
+#endif // SHARE_GC_X_X_GLOBALS_HPP
diff --git a/src/hotspot/share/gc/z/c1/zBarrierSetC1.cpp b/src/hotspot/share/gc/z/c1/zBarrierSetC1.cpp
index 93001511f88..5f2ed9c304f 100644
--- a/src/hotspot/share/gc/z/c1/zBarrierSetC1.cpp
+++ b/src/hotspot/share/gc/z/c1/zBarrierSetC1.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,8 +22,11 @@
  */
 
 #include "precompiled.hpp"
+#include "c1/c1_FrameMap.hpp"
 #include "c1/c1_LIR.hpp"
+#include "c1/c1_LIRAssembler.hpp"
 #include "c1/c1_LIRGenerator.hpp"
+#include "c1/c1_MacroAssembler.hpp"
 #include "c1/c1_CodeStubs.hpp"
 #include "gc/z/c1/zBarrierSetC1.hpp"
 #include "gc/z/zBarrierSet.hpp"
@@ -75,6 +78,7 @@ address ZLoadBarrierStubC1::runtime_stub() const {
 void ZLoadBarrierStubC1::visit(LIR_OpVisitState* visitor) {
   visitor->do_slow_case();
   visitor->do_input(_ref_addr);
+  visitor->do_input(_ref);
   visitor->do_output(_ref);
   if (_tmp->is_valid()) {
     visitor->do_temp(_tmp);
@@ -91,21 +95,78 @@ void ZLoadBarrierStubC1::print_name(outputStream* out) const {
 }
 #endif // PRODUCT
 
-class LIR_OpZLoadBarrierTest : public LIR_Op {
+ZStoreBarrierStubC1::ZStoreBarrierStubC1(LIRAccess& access,
+                                         LIR_Opr new_zaddress,
+                                         LIR_Opr new_zpointer,
+                                         LIR_Opr tmp,
+                                         bool is_atomic,
+                                         address runtime_stub) :
+    _ref_addr(access.resolved_addr()),
+    _new_zaddress(new_zaddress),
+    _new_zpointer(new_zpointer),
+    _tmp(tmp),
+    _is_atomic(is_atomic),
+    _runtime_stub(runtime_stub) {
+  assert(_ref_addr->is_address(), "Must be an address");
+}
+
+LIR_Opr ZStoreBarrierStubC1::ref_addr() const {
+  return _ref_addr;
+}
+
+LIR_Opr ZStoreBarrierStubC1::new_zaddress() const {
+  return _new_zaddress;
+}
+
+LIR_Opr ZStoreBarrierStubC1::new_zpointer() const {
+  return _new_zpointer;
+}
+
+LIR_Opr ZStoreBarrierStubC1::tmp() const {
+  return _tmp;
+}
+
+bool ZStoreBarrierStubC1::is_atomic() const {
+  return _is_atomic;
+}
+
+address ZStoreBarrierStubC1::runtime_stub() const {
+  return _runtime_stub;
+}
+
+void ZStoreBarrierStubC1::visit(LIR_OpVisitState* visitor) {
+  visitor->do_slow_case();
+  visitor->do_input(_ref_addr);
+  visitor->do_temp(_new_zpointer);
+  visitor->do_temp(_tmp);
+}
+
+void ZStoreBarrierStubC1::emit_code(LIR_Assembler* ce) {
+  ZBarrierSet::assembler()->generate_c1_store_barrier_stub(ce, this);
+}
+
+#ifndef PRODUCT
+void ZStoreBarrierStubC1::print_name(outputStream* out) const {
+  out->print("ZStoreBarrierStubC1");
+}
+#endif // PRODUCT
+
+class LIR_OpZUncolor : public LIR_Op {
 private:
   LIR_Opr _opr;
 
 public:
-  LIR_OpZLoadBarrierTest(LIR_Opr opr) :
-      LIR_Op(lir_zloadbarrier_test, LIR_OprFact::illegalOpr, NULL),
+  LIR_OpZUncolor(LIR_Opr opr) :
+      LIR_Op(),
       _opr(opr) {}
 
   virtual void visit(LIR_OpVisitState* state) {
     state->do_input(_opr);
+    state->do_output(_opr);
   }
 
   virtual void emit_code(LIR_Assembler* ce) {
-    ZBarrierSet::assembler()->generate_c1_load_barrier_test(ce, _opr);
+    ZBarrierSet::assembler()->generate_c1_uncolor(ce, _opr);
   }
 
   virtual void print_instr(outputStream* out) const {
@@ -115,7 +176,45 @@ public:
 
 #ifndef PRODUCT
   virtual const char* name() const {
-    return "lir_z_load_barrier_test";
+    return "lir_z_uncolor";
+  }
+#endif // PRODUCT
+};
+
+class LIR_OpZLoadBarrier : public LIR_Op {
+private:
+  LIR_Opr                   _opr;
+  ZLoadBarrierStubC1* const _stub;
+  const bool                _on_non_strong;
+
+public:
+  LIR_OpZLoadBarrier(LIR_Opr opr, ZLoadBarrierStubC1* stub, bool on_non_strong) :
+      LIR_Op(),
+      _opr(opr),
+      _stub(stub),
+      _on_non_strong(on_non_strong) {
+    assert(stub != nullptr, "The stub is the load barrier slow path.");
+  }
+
+  virtual void visit(LIR_OpVisitState* state) {
+    state->do_input(_opr);
+    state->do_output(_opr);
+    state->do_stub(_stub);
+  }
+
+  virtual void emit_code(LIR_Assembler* ce) {
+    ZBarrierSet::assembler()->generate_c1_load_barrier(ce, _opr, _stub, _on_non_strong);
+    ce->append_code_stub(_stub);
+  }
+
+  virtual void print_instr(outputStream* out) const {
+    _opr->print(out);
+    out->print(" ");
+  }
+
+#ifndef PRODUCT
+  virtual const char* name() const {
+    return "lir_z_load_barrier";
   }
 #endif // PRODUCT
 };
@@ -125,8 +224,10 @@ static bool barrier_needed(LIRAccess& access) {
 }
 
 ZBarrierSetC1::ZBarrierSetC1() :
-    _load_barrier_on_oop_field_preloaded_runtime_stub(NULL),
-    _load_barrier_on_weak_oop_field_preloaded_runtime_stub(NULL) {}
+    _load_barrier_on_oop_field_preloaded_runtime_stub(nullptr),
+    _load_barrier_on_weak_oop_field_preloaded_runtime_stub(nullptr),
+    _store_barrier_on_oop_field_with_healing(nullptr),
+    _store_barrier_on_oop_field_without_healing(nullptr) {}
 
 address ZBarrierSetC1::load_barrier_on_oop_field_preloaded_runtime_stub(DecoratorSet decorators) const {
   assert((decorators & ON_PHANTOM_OOP_REF) == 0, "Unsupported decorator");
@@ -139,21 +240,169 @@ address ZBarrierSetC1::load_barrier_on_oop_field_preloaded_runtime_stub(Decorato
   }
 }
 
+address ZBarrierSetC1::store_barrier_on_oop_field_runtime_stub(bool self_healing) const {
+  if (self_healing) {
+    return _store_barrier_on_oop_field_with_healing;
+  } else {
+    return _store_barrier_on_oop_field_without_healing;
+  }
+}
+
+class LIR_OpZColor : public LIR_Op {
+ friend class LIR_OpVisitState;
+
+private:
+  LIR_Opr _opr;
+
+public:
+  LIR_OpZColor(LIR_Opr opr) :
+      LIR_Op(lir_none, opr, nullptr /* info */),
+      _opr(opr) {}
+
+  virtual void visit(LIR_OpVisitState* state) {
+    state->do_input(_opr);
+    state->do_output(_opr);
+  }
+
+  virtual void emit_code(LIR_Assembler* ce) {
+    ZBarrierSet::assembler()->generate_c1_color(ce, _opr);
+  }
+
+  virtual void print_instr(outputStream* out) const {
+    _opr->print(out); out->print(" ");
+  }
+
+#ifndef PRODUCT
+  virtual const char* name() const  {
+    return "lir_z_color";
+  }
+#endif // PRODUCT
+};
+
+class LIR_OpZStoreBarrier : public LIR_Op {
+ friend class LIR_OpVisitState;
+
+private:
+  LIR_Opr       _addr;
+  LIR_Opr       _new_zaddress;
+  LIR_Opr       _new_zpointer;
+  CodeStub*     _stub;
+  CodeEmitInfo* _info;
+
+public:
+  LIR_OpZStoreBarrier(LIR_Opr addr,
+                      LIR_Opr new_zaddress,
+                      LIR_Opr new_zpointer,
+                      CodeStub* stub,
+                      CodeEmitInfo* info) :
+      LIR_Op(lir_none, new_zpointer, nullptr /* info */),
+      _addr(addr),
+      _new_zaddress(new_zaddress),
+      _new_zpointer(new_zpointer),
+      _stub(stub),
+      _info(info) {}
+
+  virtual void visit(LIR_OpVisitState* state) {
+    state->do_input(_new_zaddress);
+    state->do_input(_addr);
+
+    // Use temp registers to ensure these they use different registers.
+    state->do_temp(_addr);
+    state->do_temp(_new_zaddress);
+
+    state->do_output(_new_zpointer);
+    state->do_stub(_stub);
+
+    if (_info != nullptr) {
+      state->do_info(_info);
+    }
+  }
+
+  virtual void emit_code(LIR_Assembler* ce) {
+    const ZBarrierSetAssembler* const bs_asm =
+        (const ZBarrierSetAssembler*)BarrierSet::barrier_set()->barrier_set_assembler();
+    if (_info != nullptr) {
+      ce->add_debug_info_for_null_check_here(_info);
+    }
+    bs_asm->generate_c1_store_barrier(ce,
+                                      _addr->as_address_ptr(),
+                                      _new_zaddress,
+                                      _new_zpointer,
+                                      (ZStoreBarrierStubC1*)_stub);
+    ce->append_code_stub(_stub);
+  }
+
+  virtual void print_instr(outputStream* out) const {
+    _addr->print(out);         out->print(" ");
+    _new_zaddress->print(out); out->print(" ");
+    _new_zpointer->print(out); out->print(" ");
+  }
+
+#ifndef PRODUCT
+  virtual const char* name() const  {
+    return "lir_z_store_barrier";
+  }
+#endif // PRODUCT
+};
+
 #ifdef ASSERT
 #define __ access.gen()->lir(__FILE__, __LINE__)->
 #else
 #define __ access.gen()->lir()->
 #endif
 
-void ZBarrierSetC1::load_barrier(LIRAccess& access, LIR_Opr result) const {
-  // Fast path
-  __ append(new LIR_OpZLoadBarrierTest(result));
+LIR_Opr ZBarrierSetC1::color(LIRAccess& access, LIR_Opr ref) const {
+  // Only used from CAS where we have control over the used register
+  assert(ref->is_single_cpu(), "Should be using a register");
 
+  __ append(new LIR_OpZColor(ref));
+
+  return ref;
+}
+
+void ZBarrierSetC1::load_barrier(LIRAccess& access, LIR_Opr result) const {
   // Slow path
   const address runtime_stub = load_barrier_on_oop_field_preloaded_runtime_stub(access.decorators());
-  CodeStub* const stub = new ZLoadBarrierStubC1(access, result, runtime_stub);
-  __ branch(lir_cond_notEqual, stub);
-  __ branch_destination(stub->continuation());
+  auto stub = new ZLoadBarrierStubC1(access, result, runtime_stub);
+
+  const bool on_non_strong =
+      (access.decorators() & ON_WEAK_OOP_REF) != 0 ||
+      (access.decorators() & ON_PHANTOM_OOP_REF) != 0;
+
+  __ append(new LIR_OpZLoadBarrier(result, stub, on_non_strong));
+}
+
+LIR_Opr ZBarrierSetC1::store_barrier(LIRAccess& access, LIR_Opr new_zaddress, bool is_atomic) const {
+  LIRGenerator* gen = access.gen();
+
+  LIR_Opr new_zaddress_reg;
+  if (new_zaddress->is_single_cpu()) {
+    new_zaddress_reg = new_zaddress;
+  } else if (new_zaddress->is_constant()) {
+    new_zaddress_reg = gen->new_register(access.type());
+    gen->lir()->move(new_zaddress, new_zaddress_reg);
+  } else {
+    ShouldNotReachHere();
+  }
+
+  LIR_Opr new_zpointer = gen->new_register(T_OBJECT);
+  LIR_Opr tmp = gen->new_pointer_register();
+  ZStoreBarrierStubC1* const stub =
+      new ZStoreBarrierStubC1(access,
+                              new_zaddress_reg,
+                              new_zpointer,
+                              tmp,
+                              is_atomic,
+                              store_barrier_on_oop_field_runtime_stub(is_atomic));
+
+  __ append(new LIR_OpZStoreBarrier(access.resolved_addr(),
+                                    new_zaddress_reg,
+                                    new_zpointer,
+                                    stub,
+                                    access.access_emit_info()));
+  access.access_emit_info() = nullptr;
+
+  return new_zpointer;
 }
 
 LIR_Opr ZBarrierSetC1::resolve_address(LIRAccess& access, bool resolve_in_register) {
@@ -164,51 +413,86 @@ LIR_Opr ZBarrierSetC1::resolve_address(LIRAccess& access, bool resolve_in_regist
   return BarrierSetC1::resolve_address(access, resolve_in_register || patch_before_barrier);
 }
 
-#undef __
-
 void ZBarrierSetC1::load_at_resolved(LIRAccess& access, LIR_Opr result) {
+  if (!barrier_needed(access)) {
+    BarrierSetC1::load_at_resolved(access, result);
+    return;
+  }
+
   BarrierSetC1::load_at_resolved(access, result);
-
-  if (barrier_needed(access)) {
-    load_barrier(access, result);
-  }
+  load_barrier(access, result);
 }
 
-static void pre_load_barrier(LIRAccess& access) {
-  DecoratorSet decorators = access.decorators();
-
-  // Downgrade access to MO_UNORDERED
-  decorators = (decorators & ~MO_DECORATOR_MASK) | MO_UNORDERED;
-
-  // Remove ACCESS_WRITE
-  decorators = (decorators & ~ACCESS_WRITE);
-
-  // Generate synthetic load at
-  access.gen()->access_load_at(decorators,
-                               access.type(),
-                               access.base().item(),
-                               access.offset().opr(),
-                               access.gen()->new_register(access.type()),
-                               NULL /* patch_emit_info */,
-                               NULL /* load_emit_info */);
-}
-
-LIR_Opr ZBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) {
-  if (barrier_needed(access)) {
-    pre_load_barrier(access);
+void ZBarrierSetC1::store_at_resolved(LIRAccess& access, LIR_Opr value) {
+  if (!barrier_needed(access)) {
+    BarrierSetC1::store_at_resolved(access, value);
+    return;
   }
 
-  return BarrierSetC1::atomic_xchg_at_resolved(access, value);
+  value = store_barrier(access, value, false /* is_atomic */);
+
+  BarrierSetC1::store_at_resolved(access, value);
 }
 
 LIR_Opr ZBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value) {
-  if (barrier_needed(access)) {
-    pre_load_barrier(access);
+  if (!barrier_needed(access)) {
+    return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
   }
 
-  return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value);
+  new_value.load_item();
+  const LIR_Opr new_value_zpointer = store_barrier(access, new_value.result(), true /* is_atomic */);
+
+  cmp_value.load_item();
+  cmp_value.set_destroys_register();
+  color(access, cmp_value.result());
+
+#ifdef AMD64
+  const LIR_Opr cmp_value_opr = FrameMap::rax_oop_opr;
+#else
+  const LIR_Opr cmp_value_opr = access.gen()->new_register(T_OBJECT);
+#endif
+  access.gen()->lir()->move(cmp_value.result(), cmp_value_opr);
+
+  __ cas_obj(access.resolved_addr()->as_address_ptr()->base(),
+             cmp_value_opr,
+             new_value_zpointer,
+#ifdef RISCV
+             access.gen()->new_register(T_OBJECT),
+             access.gen()->new_register(T_OBJECT),
+             access.gen()->new_register(T_OBJECT));
+#else
+             LIR_OprFact::illegalOpr, LIR_OprFact::illegalOpr);
+#endif
+  LIR_Opr result = access.gen()->new_register(T_INT);
+  __ cmove(lir_cond_equal, LIR_OprFact::intConst(1), LIR_OprFact::intConst(0),
+           result, T_INT);
+
+  return result;
 }
 
+LIR_Opr ZBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) {
+  if (!barrier_needed(access)) {
+    return BarrierSetC1::atomic_xchg_at_resolved(access, value);
+  }
+
+  value.load_item();
+
+  LIR_Opr value_zpointer = store_barrier(access, value.result(), true /* is_atomic */);
+
+  // The parent class expects the in-parameter and out-parameter to be the same.
+  // Move the colored pointer to the expected register.
+#ifdef AMD64
+  __ xchg(access.resolved_addr(), value_zpointer, value_zpointer, LIR_OprFact::illegalOpr);
+#else
+  __ xchg(access.resolved_addr(), value_zpointer, value_zpointer, access.gen()->new_register(T_INT));
+#endif
+  __ append(new LIR_OpZUncolor(value_zpointer));
+
+  return value_zpointer;
+}
+
+#undef __
+
 class ZLoadBarrierRuntimeStubCodeGenClosure : public StubAssemblerCodeGenClosure {
 private:
   const DecoratorSet _decorators;
@@ -219,19 +503,44 @@ public:
 
   virtual OopMapSet* generate_code(StubAssembler* sasm) {
     ZBarrierSet::assembler()->generate_c1_load_barrier_runtime_stub(sasm, _decorators);
-    return NULL;
+    return nullptr;
   }
 };
 
-static address generate_c1_runtime_stub(BufferBlob* blob, DecoratorSet decorators, const char* name) {
+static address generate_c1_load_runtime_stub(BufferBlob* blob, DecoratorSet decorators, const char* name) {
   ZLoadBarrierRuntimeStubCodeGenClosure cl(decorators);
   CodeBlob* const code_blob = Runtime1::generate_blob(blob, -1 /* stub_id */, name, false /* expect_oop_map*/, &cl);
   return code_blob->code_begin();
 }
 
+class ZStoreBarrierRuntimeStubCodeGenClosure : public StubAssemblerCodeGenClosure {
+private:
+  const bool _self_healing;
+
+public:
+  ZStoreBarrierRuntimeStubCodeGenClosure(bool self_healing) :
+      _self_healing(self_healing) {}
+
+  virtual OopMapSet* generate_code(StubAssembler* sasm) {
+    ZBarrierSet::assembler()->generate_c1_store_barrier_runtime_stub(sasm, _self_healing);
+    return nullptr;
+  }
+};
+
+static address generate_c1_store_runtime_stub(BufferBlob* blob, bool self_healing, const char* name) {
+  ZStoreBarrierRuntimeStubCodeGenClosure cl(self_healing);
+  CodeBlob* const code_blob = Runtime1::generate_blob(blob, -1 /* stub_id */, name, false /* expect_oop_map*/, &cl);
+  return code_blob->code_begin();
+}
+
 void ZBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* blob) {
   _load_barrier_on_oop_field_preloaded_runtime_stub =
-    generate_c1_runtime_stub(blob, ON_STRONG_OOP_REF, "load_barrier_on_oop_field_preloaded_runtime_stub");
+    generate_c1_load_runtime_stub(blob, ON_STRONG_OOP_REF, "load_barrier_on_oop_field_preloaded_runtime_stub");
   _load_barrier_on_weak_oop_field_preloaded_runtime_stub =
-    generate_c1_runtime_stub(blob, ON_WEAK_OOP_REF, "load_barrier_on_weak_oop_field_preloaded_runtime_stub");
+    generate_c1_load_runtime_stub(blob, ON_WEAK_OOP_REF, "load_barrier_on_weak_oop_field_preloaded_runtime_stub");
+
+  _store_barrier_on_oop_field_with_healing =
+    generate_c1_store_runtime_stub(blob, true /* self_healing */, "store_barrier_on_oop_field_with_healing");
+  _store_barrier_on_oop_field_without_healing =
+    generate_c1_store_runtime_stub(blob, false /* self_healing */, "store_barrier_on_oop_field_without_healing");
 }
diff --git a/src/hotspot/share/gc/z/c1/zBarrierSetC1.hpp b/src/hotspot/share/gc/z/c1/zBarrierSetC1.hpp
index 2eec7366c53..a17c3f7628c 100644
--- a/src/hotspot/share/gc/z/c1/zBarrierSetC1.hpp
+++ b/src/hotspot/share/gc/z/c1/zBarrierSetC1.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -55,19 +55,59 @@ public:
 #endif // PRODUCT
 };
 
+class ZStoreBarrierStubC1 : public CodeStub {
+private:
+  LIR_Opr _ref_addr;
+  LIR_Opr _new_zaddress;
+  LIR_Opr _new_zpointer;
+  LIR_Opr _tmp;
+  bool    _is_atomic;
+  address _runtime_stub;
+
+public:
+  ZStoreBarrierStubC1(LIRAccess& access,
+                      LIR_Opr new_zaddress,
+                      LIR_Opr new_zpointer,
+                      LIR_Opr tmp,
+                      bool is_atomic,
+                      address runtime_stub);
+
+  LIR_Opr ref_addr() const;
+  LIR_Opr new_zaddress() const;
+  LIR_Opr new_zpointer() const;
+  LIR_Opr tmp() const;
+  bool is_atomic() const;
+  address runtime_stub() const;
+
+  virtual void emit_code(LIR_Assembler* ce);
+  virtual void visit(LIR_OpVisitState* visitor);
+
+#ifndef PRODUCT
+  virtual void print_name(outputStream* out) const;
+#endif // PRODUCT
+};
+
 class ZBarrierSetC1 : public BarrierSetC1 {
 private:
   address _load_barrier_on_oop_field_preloaded_runtime_stub;
   address _load_barrier_on_weak_oop_field_preloaded_runtime_stub;
+  address _store_barrier_on_oop_field_with_healing;
+  address _store_barrier_on_oop_field_without_healing;
 
   address load_barrier_on_oop_field_preloaded_runtime_stub(DecoratorSet decorators) const;
+  address store_barrier_on_oop_field_runtime_stub(bool self_healing) const;
+
+  LIR_Opr color(LIRAccess& access, LIR_Opr ref) const;
+
   void load_barrier(LIRAccess& access, LIR_Opr result) const;
+  LIR_Opr store_barrier(LIRAccess& access, LIR_Opr new_zaddress, bool is_atomic) const;
 
 protected:
   virtual LIR_Opr resolve_address(LIRAccess& access, bool resolve_in_register);
   virtual void load_at_resolved(LIRAccess& access, LIR_Opr result);
-  virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value);
+  virtual void store_at_resolved(LIRAccess& access, LIR_Opr value);
   virtual LIR_Opr atomic_cmpxchg_at_resolved(LIRAccess& access, LIRItem& cmp_value, LIRItem& new_value);
+  virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value);
 
 public:
   ZBarrierSetC1();
diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp
index 8d729d296e3..1903c28a6fc 100644
--- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp
+++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.cpp
@@ -22,6 +22,7 @@
  */
 
 #include "precompiled.hpp"
+#include "asm/macroAssembler.hpp"
 #include "classfile/javaClasses.hpp"
 #include "gc/z/c2/zBarrierSetC2.hpp"
 #include "gc/z/zBarrierSet.hpp"
@@ -44,63 +45,197 @@
 #include "utilities/growableArray.hpp"
 #include "utilities/macros.hpp"
 
+template<typename K, typename V, size_t _table_size>
+class ZArenaHashtable : public ResourceObj {
+  class ZArenaHashtableEntry : public ResourceObj {
+  public:
+    ZArenaHashtableEntry* _next;
+    K _key;
+    V _value;
+  };
+
+  static const size_t _table_mask = _table_size - 1;
+
+  Arena* _arena;
+  ZArenaHashtableEntry* _table[_table_size];
+
+public:
+  class Iterator {
+    ZArenaHashtable* _table;
+    ZArenaHashtableEntry* _current_entry;
+    size_t _current_index;
+
+  public:
+    Iterator(ZArenaHashtable* table) :
+        _table(table),
+        _current_entry(table->_table[0]),
+        _current_index(0) {
+      if (_current_entry == nullptr) {
+        next();
+      }
+    }
+
+    bool has_next() { return _current_entry != nullptr; }
+    K key()         { return _current_entry->_key; }
+    V value()       { return _current_entry->_value; }
+
+    void next() {
+      if (_current_entry != nullptr) {
+        _current_entry = _current_entry->_next;
+      }
+      while (_current_entry == nullptr && ++_current_index < _table_size) {
+        _current_entry = _table->_table[_current_index];
+      }
+    }
+  };
+
+  ZArenaHashtable(Arena* arena) :
+      _arena(arena),
+      _table() {
+    Copy::zero_to_bytes(&_table, sizeof(_table));
+  }
+
+  void add(K key, V value) {
+    ZArenaHashtableEntry* entry = new (_arena) ZArenaHashtableEntry();
+    entry->_key = key;
+    entry->_value = value;
+    entry->_next = _table[key & _table_mask];
+    _table[key & _table_mask] = entry;
+  }
+
+  V* get(K key) const {
+    for (ZArenaHashtableEntry* e = _table[key & _table_mask]; e != nullptr; e = e->_next) {
+      if (e->_key == key) {
+        return &(e->_value);
+      }
+    }
+    return nullptr;
+  }
+
+  Iterator iterator() {
+    return Iterator(this);
+  }
+};
+
+typedef ZArenaHashtable<intptr_t, bool, 4> ZOffsetTable;
+
 class ZBarrierSetC2State : public ArenaObj {
 private:
-  GrowableArray<ZLoadBarrierStubC2*>* _stubs;
-  Node_Array                          _live;
+  GrowableArray<ZBarrierStubC2*>* _stubs;
+  Node_Array                      _live;
+  int                             _trampoline_stubs_count;
+  int                             _stubs_start_offset;
 
 public:
   ZBarrierSetC2State(Arena* arena) :
-    _stubs(new (arena) GrowableArray<ZLoadBarrierStubC2*>(arena, 8,  0, NULL)),
-    _live(arena) {}
+      _stubs(new (arena) GrowableArray<ZBarrierStubC2*>(arena, 8,  0, nullptr)),
+      _live(arena),
+      _trampoline_stubs_count(0),
+      _stubs_start_offset(0) {}
 
-  GrowableArray<ZLoadBarrierStubC2*>* stubs() {
+  GrowableArray<ZBarrierStubC2*>* stubs() {
     return _stubs;
   }
 
   RegMask* live(const Node* node) {
     if (!node->is_Mach()) {
       // Don't need liveness for non-MachNodes
-      return NULL;
+      return nullptr;
     }
 
     const MachNode* const mach = node->as_Mach();
-    if (mach->barrier_data() == ZLoadBarrierElided) {
+    if (mach->barrier_data() == ZBarrierElided) {
       // Don't need liveness data for nodes without barriers
-      return NULL;
+      return nullptr;
     }
 
     RegMask* live = (RegMask*)_live[node->_idx];
-    if (live == NULL) {
+    if (live == nullptr) {
       live = new (Compile::current()->comp_arena()->AmallocWords(sizeof(RegMask))) RegMask();
       _live.map(node->_idx, (Node*)live);
     }
 
     return live;
   }
+
+  void inc_trampoline_stubs_count() {
+    assert(_trampoline_stubs_count != INT_MAX, "Overflow");
+    ++_trampoline_stubs_count;
+  }
+
+  int trampoline_stubs_count() {
+    return _trampoline_stubs_count;
+  }
+
+  void set_stubs_start_offset(int offset) {
+    _stubs_start_offset = offset;
+  }
+
+  int stubs_start_offset() {
+    return _stubs_start_offset;
+  }
 };
 
 static ZBarrierSetC2State* barrier_set_state() {
   return reinterpret_cast<ZBarrierSetC2State*>(Compile::current()->barrier_set_state());
 }
 
-ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data) {
-  ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref, tmp, barrier_data);
+void ZBarrierStubC2::register_stub(ZBarrierStubC2* stub) {
   if (!Compile::current()->output()->in_scratch_emit_size()) {
     barrier_set_state()->stubs()->append(stub);
   }
+}
+
+void ZBarrierStubC2::inc_trampoline_stubs_count() {
+  if (!Compile::current()->output()->in_scratch_emit_size()) {
+    barrier_set_state()->inc_trampoline_stubs_count();
+  }
+}
+
+int ZBarrierStubC2::trampoline_stubs_count() {
+  return barrier_set_state()->trampoline_stubs_count();
+}
+
+int ZBarrierStubC2::stubs_start_offset() {
+  return barrier_set_state()->stubs_start_offset();
+}
+
+ZBarrierStubC2::ZBarrierStubC2(const MachNode* node) :
+    _node(node),
+    _entry(),
+    _continuation() {}
+
+Register ZBarrierStubC2::result() const {
+  return noreg;
+}
+
+RegMask& ZBarrierStubC2::live() const {
+  return *barrier_set_state()->live(_node);
+}
+
+Label* ZBarrierStubC2::entry() {
+  // The _entry will never be bound when in_scratch_emit_size() is true.
+  // However, we still need to return a label that is not bound now, but
+  // will eventually be bound. Any eventually bound label will do, as it
+  // will only act as a placeholder, so we return the _continuation label.
+  return Compile::current()->output()->in_scratch_emit_size() ? &_continuation : &_entry;
+}
+
+Label* ZBarrierStubC2::continuation() {
+  return &_continuation;
+}
+
+ZLoadBarrierStubC2* ZLoadBarrierStubC2::create(const MachNode* node, Address ref_addr, Register ref) {
+  ZLoadBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZLoadBarrierStubC2(node, ref_addr, ref);
+  register_stub(stub);
 
   return stub;
 }
 
-ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data) :
-    _node(node),
+ZLoadBarrierStubC2::ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref) :
+    ZBarrierStubC2(node),
     _ref_addr(ref_addr),
-    _ref(ref),
-    _tmp(tmp),
-    _barrier_data(barrier_data),
-    _entry(),
-    _continuation() {
+    _ref(ref) {
   assert_different_registers(ref, ref_addr.base());
   assert_different_registers(ref, ref_addr.index());
 }
@@ -113,43 +248,74 @@ Register ZLoadBarrierStubC2::ref() const {
   return _ref;
 }
 
-Register ZLoadBarrierStubC2::tmp() const {
-  return _tmp;
+Register ZLoadBarrierStubC2::result() const {
+  return ref();
 }
 
 address ZLoadBarrierStubC2::slow_path() const {
+  const uint8_t barrier_data = _node->barrier_data();
   DecoratorSet decorators = DECORATORS_NONE;
-  if (_barrier_data & ZLoadBarrierStrong) {
+  if (barrier_data & ZBarrierStrong) {
     decorators |= ON_STRONG_OOP_REF;
   }
-  if (_barrier_data & ZLoadBarrierWeak) {
+  if (barrier_data & ZBarrierWeak) {
     decorators |= ON_WEAK_OOP_REF;
   }
-  if (_barrier_data & ZLoadBarrierPhantom) {
+  if (barrier_data & ZBarrierPhantom) {
     decorators |= ON_PHANTOM_OOP_REF;
   }
-  if (_barrier_data & ZLoadBarrierNoKeepalive) {
+  if (barrier_data & ZBarrierNoKeepalive) {
     decorators |= AS_NO_KEEPALIVE;
   }
   return ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(decorators);
 }
 
-RegMask& ZLoadBarrierStubC2::live() const {
-  RegMask* mask = barrier_set_state()->live(_node);
-  assert(mask != NULL, "must be mach-node with barrier");
-  return *mask;
+void ZLoadBarrierStubC2::emit_code(MacroAssembler& masm) {
+  ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, static_cast<ZLoadBarrierStubC2*>(this));
 }
 
-Label* ZLoadBarrierStubC2::entry() {
-  // The _entry will never be bound when in_scratch_emit_size() is true.
-  // However, we still need to return a label that is not bound now, but
-  // will eventually be bound. Any label will do, as it will only act as
-  // a placeholder, so we return the _continuation label.
-  return Compile::current()->output()->in_scratch_emit_size() ? &_continuation : &_entry;
+ZStoreBarrierStubC2* ZStoreBarrierStubC2::create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic) {
+  ZStoreBarrierStubC2* const stub = new (Compile::current()->comp_arena()) ZStoreBarrierStubC2(node, ref_addr, new_zaddress, new_zpointer, is_native, is_atomic);
+  register_stub(stub);
+
+  return stub;
 }
 
-Label* ZLoadBarrierStubC2::continuation() {
-  return &_continuation;
+ZStoreBarrierStubC2::ZStoreBarrierStubC2(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic) :
+    ZBarrierStubC2(node),
+    _ref_addr(ref_addr),
+    _new_zaddress(new_zaddress),
+    _new_zpointer(new_zpointer),
+    _is_native(is_native),
+    _is_atomic(is_atomic) {
+}
+
+Address ZStoreBarrierStubC2::ref_addr() const {
+  return _ref_addr;
+}
+
+Register ZStoreBarrierStubC2::new_zaddress() const {
+  return _new_zaddress;
+}
+
+Register ZStoreBarrierStubC2::new_zpointer() const {
+  return _new_zpointer;
+}
+
+bool ZStoreBarrierStubC2::is_native() const {
+  return _is_native;
+}
+
+bool ZStoreBarrierStubC2::is_atomic() const {
+  return _is_atomic;
+}
+
+Register ZStoreBarrierStubC2::result() const {
+  return noreg;
+}
+
+void ZStoreBarrierStubC2::emit_code(MacroAssembler& masm) {
+  ZBarrierSet::assembler()->generate_c2_store_barrier_stub(&masm, static_cast<ZStoreBarrierStubC2*>(this));
 }
 
 void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
@@ -157,22 +323,23 @@ void* ZBarrierSetC2::create_barrier_state(Arena* comp_arena) const {
 }
 
 void ZBarrierSetC2::late_barrier_analysis() const {
-  analyze_dominating_barriers();
   compute_liveness_at_stubs();
+  analyze_dominating_barriers();
 }
 
 void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
   MacroAssembler masm(&cb);
-  GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
+  GrowableArray<ZBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
+  barrier_set_state()->set_stubs_start_offset(masm.offset());
 
   for (int i = 0; i < stubs->length(); i++) {
     // Make sure there is enough space in the code buffer
-    if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == NULL) {
+    if (cb.insts()->maybe_expand_to_ensure_remaining(PhaseOutput::MAX_inst_size) && cb.blob() == nullptr) {
       ciEnv::current()->record_failure("CodeCache is full");
       return;
     }
 
-    ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
+    stubs->at(i)->emit_code(masm);
   }
 
   masm.flush();
@@ -181,13 +348,13 @@ void ZBarrierSetC2::emit_stubs(CodeBuffer& cb) const {
 int ZBarrierSetC2::estimate_stub_size() const {
   Compile* const C = Compile::current();
   BufferBlob* const blob = C->output()->scratch_buffer_blob();
-  GrowableArray<ZLoadBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
+  GrowableArray<ZBarrierStubC2*>* const stubs = barrier_set_state()->stubs();
   int size = 0;
 
   for (int i = 0; i < stubs->length(); i++) {
     CodeBuffer cb(blob->content_begin(), (address)C->output()->scratch_locs_memory() - blob->content_begin());
     MacroAssembler masm(&cb);
-    ZBarrierSet::assembler()->generate_c2_load_barrier_stub(&masm, stubs->at(i));
+    stubs->at(i)->emit_code(masm);
     size += cb.insts_size();
   }
 
@@ -195,23 +362,39 @@ int ZBarrierSetC2::estimate_stub_size() const {
 }
 
 static void set_barrier_data(C2Access& access) {
-  if (ZBarrierSet::barrier_needed(access.decorators(), access.type())) {
-    uint8_t barrier_data = 0;
-
-    if (access.decorators() & ON_PHANTOM_OOP_REF) {
-      barrier_data |= ZLoadBarrierPhantom;
-    } else if (access.decorators() & ON_WEAK_OOP_REF) {
-      barrier_data |= ZLoadBarrierWeak;
-    } else {
-      barrier_data |= ZLoadBarrierStrong;
-    }
-
-    if (access.decorators() & AS_NO_KEEPALIVE) {
-      barrier_data |= ZLoadBarrierNoKeepalive;
-    }
-
-    access.set_barrier_data(barrier_data);
+  if (!ZBarrierSet::barrier_needed(access.decorators(), access.type())) {
+    return;
   }
+
+  if (access.decorators() & C2_TIGHTLY_COUPLED_ALLOC) {
+    access.set_barrier_data(ZBarrierElided);
+    return;
+  }
+
+  uint8_t barrier_data = 0;
+
+  if (access.decorators() & ON_PHANTOM_OOP_REF) {
+    barrier_data |= ZBarrierPhantom;
+  } else if (access.decorators() & ON_WEAK_OOP_REF) {
+    barrier_data |= ZBarrierWeak;
+  } else {
+    barrier_data |= ZBarrierStrong;
+  }
+
+  if (access.decorators() & IN_NATIVE) {
+    barrier_data |= ZBarrierNative;
+  }
+
+  if (access.decorators() & AS_NO_KEEPALIVE) {
+    barrier_data |= ZBarrierNoKeepalive;
+  }
+
+  access.set_barrier_data(barrier_data);
+}
+
+Node* ZBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val) const {
+  set_barrier_data(access);
+  return BarrierSetC2::store_at_resolved(access, val);
 }
 
 Node* ZBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const {
@@ -252,16 +435,16 @@ bool ZBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_coupled_alloc,
 // This TypeFunc assumes a 64bit system
 static const TypeFunc* clone_type() {
   // Create input type (domain)
-  const Type** domain_fields = TypeTuple::fields(4);
+  const Type** const domain_fields = TypeTuple::fields(4);
   domain_fields[TypeFunc::Parms + 0] = TypeInstPtr::NOTNULL;  // src
   domain_fields[TypeFunc::Parms + 1] = TypeInstPtr::NOTNULL;  // dst
   domain_fields[TypeFunc::Parms + 2] = TypeLong::LONG;        // size lower
   domain_fields[TypeFunc::Parms + 3] = Type::HALF;            // size upper
-  const TypeTuple* domain = TypeTuple::make(TypeFunc::Parms + 4, domain_fields);
+  const TypeTuple* const domain = TypeTuple::make(TypeFunc::Parms + 4, domain_fields);
 
   // Create result type (range)
-  const Type** range_fields = TypeTuple::fields(0);
-  const TypeTuple* range = TypeTuple::make(TypeFunc::Parms + 0, range_fields);
+  const Type** const range_fields = TypeTuple::fields(0);
+  const TypeTuple* const range = TypeTuple::make(TypeFunc::Parms + 0, range_fields);
 
   return TypeFunc::make(domain, range);
 }
@@ -270,9 +453,9 @@ static const TypeFunc* clone_type() {
 
 void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* ac) const {
   Node* const src = ac->in(ArrayCopyNode::Src);
-  const TypeAryPtr* ary_ptr = src->get_ptr_type()->isa_aryptr();
+  const TypeAryPtr* const ary_ptr = src->get_ptr_type()->isa_aryptr();
 
-  if (ac->is_clone_array() && ary_ptr != NULL) {
+  if (ac->is_clone_array() && ary_ptr != nullptr) {
     BasicType bt = ary_ptr->elem()->array_element_basic_type();
     if (is_reference_type(bt)) {
       // Clone object array
@@ -282,11 +465,11 @@ void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* a
       bt = T_LONG;
     }
 
-    Node* ctrl = ac->in(TypeFunc::Control);
-    Node* mem = ac->in(TypeFunc::Memory);
-    Node* src = ac->in(ArrayCopyNode::Src);
+    Node* const ctrl = ac->in(TypeFunc::Control);
+    Node* const mem = ac->in(TypeFunc::Memory);
+    Node* const src = ac->in(ArrayCopyNode::Src);
     Node* src_offset = ac->in(ArrayCopyNode::SrcPos);
-    Node* dest = ac->in(ArrayCopyNode::Dest);
+    Node* const dest = ac->in(ArrayCopyNode::Dest);
     Node* dest_offset = ac->in(ArrayCopyNode::DestPos);
     Node* length = ac->in(ArrayCopyNode::Length);
 
@@ -296,7 +479,7 @@ void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* a
       // to the first element in the array when cloning object arrays. Otherwise, load
       // barriers are applied to parts of the header. Also adjust the length accordingly.
       assert(src_offset == dest_offset, "should be equal");
-      jlong offset = src_offset->get_long();
+      const jlong offset = src_offset->get_long();
       if (offset != arrayOopDesc::base_offset_in_bytes(T_OBJECT)) {
         assert(!UseCompressedClassPointers, "should only happen without compressed class pointers");
         assert((arrayOopDesc::base_offset_in_bytes(T_OBJECT) - offset) == BytesPerLong, "unexpected offset");
@@ -305,16 +488,16 @@ void ZBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCopyNode* a
         dest_offset = src_offset;
       }
     }
-    Node* payload_src = phase->basic_plus_adr(src, src_offset);
-    Node* payload_dst = phase->basic_plus_adr(dest, dest_offset);
+    Node* const payload_src = phase->basic_plus_adr(src, src_offset);
+    Node* const payload_dst = phase->basic_plus_adr(dest, dest_offset);
 
-    const char* copyfunc_name = "arraycopy";
-    address     copyfunc_addr = phase->basictype2arraycopy(bt, NULL, NULL, true, copyfunc_name, true);
+    const char*   copyfunc_name = "arraycopy";
+    const address copyfunc_addr = phase->basictype2arraycopy(bt, nullptr, nullptr, true, copyfunc_name, true);
 
-    const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
-    const TypeFunc* call_type = OptoRuntime::fast_arraycopy_Type();
+    const TypePtr* const raw_adr_type = TypeRawPtr::BOTTOM;
+    const TypeFunc* const call_type = OptoRuntime::fast_arraycopy_Type();
 
-    Node* call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
+    Node* const call = phase->make_leaf_call(ctrl, mem, call_type, copyfunc_addr, copyfunc_name, raw_adr_type, payload_src, payload_dst, length XTOP);
     phase->transform_later(call);
 
     phase->igvn().replace_node(ac, call);
@@ -378,92 +561,213 @@ static uint block_index(const Block* block, const Node* node) {
   return 0;
 }
 
-void ZBarrierSetC2::analyze_dominating_barriers() const {
-  ResourceMark rm;
-  Compile* const C = Compile::current();
-  PhaseCFG* const cfg = C->cfg();
-  Block_List worklist;
-  Node_List mem_ops;
-  Node_List barrier_loads;
-
-  // Step 1 - Find accesses, and track them in lists
-  for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
-    const Block* const block = cfg->get_block(i);
-    for (uint j = 0; j < block->number_of_nodes(); ++j) {
-      const Node* const node = block->get_node(j);
-      if (!node->is_Mach()) {
-        continue;
+// Look through various node aliases
+static const Node* look_through_node(const Node* node) {
+  while (node != nullptr) {
+    const Node* new_node = node;
+    if (node->is_Mach()) {
+      const MachNode* const node_mach = node->as_Mach();
+      if (node_mach->ideal_Opcode() == Op_CheckCastPP) {
+        new_node = node->in(1);
       }
-
-      MachNode* const mach = node->as_Mach();
-      switch (mach->ideal_Opcode()) {
-      case Op_LoadP:
-        if ((mach->barrier_data() & ZLoadBarrierStrong) != 0) {
-          barrier_loads.push(mach);
-        }
-        if ((mach->barrier_data() & (ZLoadBarrierStrong | ZLoadBarrierNoKeepalive)) ==
-            ZLoadBarrierStrong) {
-          mem_ops.push(mach);
-        }
-        break;
-      case Op_CompareAndExchangeP:
-      case Op_CompareAndSwapP:
-      case Op_GetAndSetP:
-        if ((mach->barrier_data() & ZLoadBarrierStrong) != 0) {
-          barrier_loads.push(mach);
-        }
-      case Op_StoreP:
-        mem_ops.push(mach);
-        break;
-
-      default:
-        break;
+      if (node_mach->is_SpillCopy()) {
+        new_node = node->in(1);
       }
     }
+    if (new_node == node || new_node == nullptr) {
+      break;
+    } else {
+      node = new_node;
+    }
   }
 
-  // Step 2 - Find dominating accesses for each load
-  for (uint i = 0; i < barrier_loads.size(); i++) {
-    MachNode* const load = barrier_loads.at(i)->as_Mach();
-    const TypePtr* load_adr_type = NULL;
-    intptr_t load_offset = 0;
-    const Node* const load_obj = load->get_base_and_disp(load_offset, load_adr_type);
-    Block* const load_block = cfg->get_block_for_node(load);
-    const uint load_index = block_index(load_block, load);
+  return node;
+}
 
-    for (uint j = 0; j < mem_ops.size(); j++) {
-      MachNode* mem = mem_ops.at(j)->as_Mach();
-      const TypePtr* mem_adr_type = NULL;
-      intptr_t mem_offset = 0;
-      const Node* mem_obj = mem->get_base_and_disp(mem_offset, mem_adr_type);
-      Block* mem_block = cfg->get_block_for_node(mem);
-      uint mem_index = block_index(mem_block, mem);
+// Whether the given offset is undefined.
+static bool is_undefined(intptr_t offset) {
+  return offset == Type::OffsetTop;
+}
 
-      if (load_obj == NodeSentinel || mem_obj == NodeSentinel ||
-          load_obj == NULL || mem_obj == NULL ||
-          load_offset < 0 || mem_offset < 0) {
+// Whether the given offset is unknown.
+static bool is_unknown(intptr_t offset) {
+  return offset == Type::OffsetBot;
+}
+
+// Whether the given offset is concrete (defined and compile-time known).
+static bool is_concrete(intptr_t offset) {
+  return !is_undefined(offset) && !is_unknown(offset);
+}
+
+// Compute base + offset components of the memory address accessed by mach.
+// Return a node representing the base address, or null if the base cannot be
+// found or the offset is undefined or a concrete negative value. If a non-null
+// base is returned, the offset is a concrete, nonnegative value or unknown.
+static const Node* get_base_and_offset(const MachNode* mach, intptr_t& offset) {
+  const TypePtr* adr_type = nullptr;
+  offset = 0;
+  const Node* base = mach->get_base_and_disp(offset, adr_type);
+
+  if (base == nullptr || base == NodeSentinel) {
+    return nullptr;
+  }
+
+  if (offset == 0 && base->is_Mach() && base->as_Mach()->ideal_Opcode() == Op_AddP) {
+    // The memory address is computed by 'base' and fed to 'mach' via an
+    // indirect memory operand (indicated by offset == 0). The ultimate base and
+    // offset can be fetched directly from the inputs and Ideal type of 'base'.
+    offset = base->bottom_type()->isa_oopptr()->offset();
+    // Even if 'base' is not an Ideal AddP node anymore, Matcher::ReduceInst()
+    // guarantees that the base address is still available at the same slot.
+    base = base->in(AddPNode::Base);
+    assert(base != nullptr, "");
+  }
+
+  if (is_undefined(offset) || (is_concrete(offset) && offset < 0)) {
+    return nullptr;
+  }
+
+  return look_through_node(base);
+}
+
+// Whether a phi node corresponds to an array allocation.
+// This test is incomplete: in some edge cases, it might return false even
+// though the node does correspond to an array allocation.
+static bool is_array_allocation(const Node* phi) {
+  precond(phi->is_Phi());
+  // Check whether phi has a successor cast (CheckCastPP) to Java array pointer,
+  // possibly below spill copies and other cast nodes. Limit the exploration to
+  // a single path from the phi node consisting of these node types.
+  const Node* current = phi;
+  while (true) {
+    const Node* next = nullptr;
+    for (DUIterator_Fast imax, i = current->fast_outs(imax); i < imax; i++) {
+      if (!current->fast_out(i)->isa_Mach()) {
         continue;
       }
-
-      if (mem_obj != load_obj || mem_offset != load_offset) {
-        // Not the same addresses, not a candidate
-        continue;
-      }
-
-      if (load_block == mem_block) {
-        // Earlier accesses in the same block
-        if (mem_index < load_index && !block_has_safepoint(mem_block, mem_index + 1, load_index)) {
-          load->set_barrier_data(ZLoadBarrierElided);
+      const MachNode* succ = current->fast_out(i)->as_Mach();
+      if (succ->ideal_Opcode() == Op_CheckCastPP) {
+        if (succ->get_ptr_type()->isa_aryptr()) {
+          // Cast to Java array pointer: phi corresponds to an array allocation.
+          return true;
         }
-      } else if (mem_block->dominates(load_block)) {
+        // Other cast: record as candidate for further exploration.
+        next = succ;
+      } else if (succ->is_SpillCopy() && next == nullptr) {
+        // Spill copy, and no better candidate found: record as candidate.
+        next = succ;
+      }
+    }
+    if (next == nullptr) {
+      // No evidence found that phi corresponds to an array allocation, and no
+      // candidates available to continue exploring.
+      return false;
+    }
+    // Continue exploring from the best candidate found.
+    current = next;
+  }
+  ShouldNotReachHere();
+}
+
+// Match the phi node that connects a TLAB allocation fast path with its slowpath
+static bool is_allocation(const Node* node) {
+  if (node->req() != 3) {
+    return false;
+  }
+  const Node* const fast_node = node->in(2);
+  if (!fast_node->is_Mach()) {
+    return false;
+  }
+  const MachNode* const fast_mach = fast_node->as_Mach();
+  if (fast_mach->ideal_Opcode() != Op_LoadP) {
+    return false;
+  }
+  const TypePtr* const adr_type = nullptr;
+  intptr_t offset;
+  const Node* const base = get_base_and_offset(fast_mach, offset);
+  if (base == nullptr || !base->is_Mach() || !is_concrete(offset)) {
+    return false;
+  }
+  const MachNode* const base_mach = base->as_Mach();
+  if (base_mach->ideal_Opcode() != Op_ThreadLocal) {
+    return false;
+  }
+  return offset == in_bytes(Thread::tlab_top_offset());
+}
+
+static void elide_mach_barrier(MachNode* mach) {
+  mach->set_barrier_data(ZBarrierElided);
+}
+
+void ZBarrierSetC2::analyze_dominating_barriers_impl(Node_List& accesses, Node_List& access_dominators) const {
+  Compile* const C = Compile::current();
+  PhaseCFG* const cfg = C->cfg();
+
+  for (uint i = 0; i < accesses.size(); i++) {
+    MachNode* const access = accesses.at(i)->as_Mach();
+    intptr_t access_offset;
+    const Node* const access_obj = get_base_and_offset(access, access_offset);
+    Block* const access_block = cfg->get_block_for_node(access);
+    const uint access_index = block_index(access_block, access);
+
+    if (access_obj == nullptr) {
+      // No information available
+      continue;
+    }
+
+    for (uint j = 0; j < access_dominators.size(); j++) {
+     const  Node* const mem = access_dominators.at(j);
+      if (mem->is_Phi()) {
+        // Allocation node
+        if (mem != access_obj) {
+          continue;
+        }
+        if (is_unknown(access_offset) && !is_array_allocation(mem)) {
+          // The accessed address has an unknown offset, but the allocated
+          // object cannot be determined to be an array. Avoid eliding in this
+          // case, to be on the safe side.
+          continue;
+        }
+        assert((is_concrete(access_offset) && access_offset >= 0) || (is_unknown(access_offset) && is_array_allocation(mem)),
+               "candidate allocation-dominated access offsets must be either concrete and nonnegative, or unknown (for array allocations only)");
+      } else {
+        // Access node
+        const MachNode* const mem_mach = mem->as_Mach();
+        intptr_t mem_offset;
+        const Node* const mem_obj = get_base_and_offset(mem_mach, mem_offset);
+
+        if (mem_obj == nullptr ||
+            !is_concrete(access_offset) ||
+            !is_concrete(mem_offset)) {
+          // No information available
+          continue;
+        }
+
+        if (mem_obj != access_obj || mem_offset != access_offset) {
+          // Not the same addresses, not a candidate
+          continue;
+        }
+        assert(is_concrete(access_offset) && access_offset >= 0,
+               "candidate non-allocation-dominated access offsets must be concrete and nonnegative");
+      }
+
+      Block* mem_block = cfg->get_block_for_node(mem);
+      const uint mem_index = block_index(mem_block, mem);
+
+      if (access_block == mem_block) {
+        // Earlier accesses in the same block
+        if (mem_index < access_index && !block_has_safepoint(mem_block, mem_index + 1, access_index)) {
+          elide_mach_barrier(access);
+        }
+      } else if (mem_block->dominates(access_block)) {
         // Dominating block? Look around for safepoints
         ResourceMark rm;
         Block_List stack;
         VectorSet visited;
-        stack.push(load_block);
-        bool safepoint_found = block_has_safepoint(load_block);
+        stack.push(access_block);
+        bool safepoint_found = block_has_safepoint(access_block);
         while (!safepoint_found && stack.size() > 0) {
-          Block* block = stack.pop();
+          const Block* const block = stack.pop();
           if (visited.test_set(block->_pre_order)) {
             continue;
           }
@@ -477,19 +781,93 @@ void ZBarrierSetC2::analyze_dominating_barriers() const {
 
           // Push predecessor blocks
           for (uint p = 1; p < block->num_preds(); ++p) {
-            Block* pred = cfg->get_block_for_node(block->pred(p));
+            Block* const pred = cfg->get_block_for_node(block->pred(p));
             stack.push(pred);
           }
         }
 
         if (!safepoint_found) {
-          load->set_barrier_data(ZLoadBarrierElided);
+          elide_mach_barrier(access);
         }
       }
     }
   }
 }
 
+void ZBarrierSetC2::analyze_dominating_barriers() const {
+  ResourceMark rm;
+  Compile* const C = Compile::current();
+  PhaseCFG* const cfg = C->cfg();
+
+  Node_List loads;
+  Node_List load_dominators;
+
+  Node_List stores;
+  Node_List store_dominators;
+
+  Node_List atomics;
+  Node_List atomic_dominators;
+
+  // Step 1 - Find accesses and allocations, and track them in lists
+  for (uint i = 0; i < cfg->number_of_blocks(); ++i) {
+    const Block* const block = cfg->get_block(i);
+    for (uint j = 0; j < block->number_of_nodes(); ++j) {
+      Node* const node = block->get_node(j);
+      if (node->is_Phi()) {
+        if (is_allocation(node)) {
+          load_dominators.push(node);
+          store_dominators.push(node);
+          // An allocation can't be considered to "dominate" an atomic operation.
+          // For example a CAS requires the memory location to be store-good.
+          // When you have a dominating store or atomic instruction, that is
+          // indeed ensured to be the case. However, as for allocations, the
+          // initialized memory location could be raw null, which isn't store-good.
+        }
+        continue;
+      } else if (!node->is_Mach()) {
+        continue;
+      }
+
+      MachNode* const mach = node->as_Mach();
+      switch (mach->ideal_Opcode()) {
+      case Op_LoadP:
+        if ((mach->barrier_data() & ZBarrierStrong) != 0 &&
+            (mach->barrier_data() & ZBarrierNoKeepalive) == 0) {
+          loads.push(mach);
+          load_dominators.push(mach);
+        }
+        break;
+      case Op_StoreP:
+        if (mach->barrier_data() != 0) {
+          stores.push(mach);
+          load_dominators.push(mach);
+          store_dominators.push(mach);
+          atomic_dominators.push(mach);
+        }
+        break;
+      case Op_CompareAndExchangeP:
+      case Op_CompareAndSwapP:
+      case Op_GetAndSetP:
+        if (mach->barrier_data() != 0) {
+          atomics.push(mach);
+          load_dominators.push(mach);
+          store_dominators.push(mach);
+          atomic_dominators.push(mach);
+        }
+        break;
+
+      default:
+        break;
+      }
+    }
+  }
+
+  // Step 2 - Find dominating accesses or allocations for each access
+  analyze_dominating_barriers_impl(loads, load_dominators);
+  analyze_dominating_barriers_impl(stores, store_dominators);
+  analyze_dominating_barriers_impl(atomics, atomic_dominators);
+}
+
 // == Reduced spilling optimization ==
 
 void ZBarrierSetC2::compute_liveness_at_stubs() const {
@@ -547,7 +925,7 @@ void ZBarrierSetC2::compute_liveness_at_stubs() const {
 
       // If this node tracks liveness, update it
       RegMask* const regs = barrier_set_state()->live(node);
-      if (regs != NULL) {
+      if (regs != nullptr) {
         regs->OR(new_live);
       }
     }
@@ -565,19 +943,39 @@ void ZBarrierSetC2::compute_liveness_at_stubs() const {
   }
 }
 
+void ZBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const {
+  eliminate_gc_barrier_data(node);
+}
+
+void ZBarrierSetC2::eliminate_gc_barrier_data(Node* node) const {
+  if (node->is_LoadStore()) {
+    LoadStoreNode* loadstore = node->as_LoadStore();
+    loadstore->set_barrier_data(ZBarrierElided);
+  } else if (node->is_Mem()) {
+    MemNode* mem = node->as_Mem();
+    mem->set_barrier_data(ZBarrierElided);
+  }
+}
+
 #ifndef PRODUCT
 void ZBarrierSetC2::dump_barrier_data(const MachNode* mach, outputStream* st) const {
-  if ((mach->barrier_data() & ZLoadBarrierStrong) != 0) {
+  if ((mach->barrier_data() & ZBarrierStrong) != 0) {
     st->print("strong ");
   }
-  if ((mach->barrier_data() & ZLoadBarrierWeak) != 0) {
+  if ((mach->barrier_data() & ZBarrierWeak) != 0) {
     st->print("weak ");
   }
-  if ((mach->barrier_data() & ZLoadBarrierPhantom) != 0) {
+  if ((mach->barrier_data() & ZBarrierPhantom) != 0) {
     st->print("phantom ");
   }
-  if ((mach->barrier_data() & ZLoadBarrierNoKeepalive) != 0) {
+  if ((mach->barrier_data() & ZBarrierNoKeepalive) != 0) {
     st->print("nokeepalive ");
   }
+  if ((mach->barrier_data() & ZBarrierNative) != 0) {
+    st->print("native ");
+  }
+  if ((mach->barrier_data() & ZBarrierElided) != 0) {
+    st->print("elided ");
+  }
 }
 #endif // !PRODUCT
diff --git a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp
index 144ffb26bc4..a0f29fbc510 100644
--- a/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp
+++ b/src/hotspot/share/gc/z/c2/zBarrierSetC2.hpp
@@ -29,42 +29,91 @@
 #include "opto/node.hpp"
 #include "utilities/growableArray.hpp"
 
-const uint8_t ZLoadBarrierElided      = 0;
-const uint8_t ZLoadBarrierStrong      = 1;
-const uint8_t ZLoadBarrierWeak        = 2;
-const uint8_t ZLoadBarrierPhantom     = 4;
-const uint8_t ZLoadBarrierNoKeepalive = 8;
+const uint8_t ZBarrierStrong      =  1;
+const uint8_t ZBarrierWeak        =  2;
+const uint8_t ZBarrierPhantom     =  4;
+const uint8_t ZBarrierNoKeepalive =  8;
+const uint8_t ZBarrierNative      = 16;
+const uint8_t ZBarrierElided      = 32;
 
-class ZLoadBarrierStubC2 : public ArenaObj {
-private:
+class Block;
+class MachNode;
+
+class MacroAssembler;
+
+class ZBarrierStubC2 : public ArenaObj {
+protected:
   const MachNode* _node;
-  const Address   _ref_addr;
-  const Register  _ref;
-  const Register  _tmp;
-  const uint8_t   _barrier_data;
   Label           _entry;
   Label           _continuation;
 
-  ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data);
+static void register_stub(ZBarrierStubC2* stub);
+static void inc_trampoline_stubs_count();
+static int trampoline_stubs_count();
+static int stubs_start_offset();
 
 public:
-  static ZLoadBarrierStubC2* create(const MachNode* node, Address ref_addr, Register ref, Register tmp, uint8_t barrier_data);
+  ZBarrierStubC2(const MachNode* node);
 
-  Address ref_addr() const;
-  Register ref() const;
-  Register tmp() const;
-  address slow_path() const;
   RegMask& live() const;
   Label* entry();
   Label* continuation();
+
+  virtual Register result() const = 0;
+  virtual void emit_code(MacroAssembler& masm) = 0;
+};
+
+class ZLoadBarrierStubC2 : public ZBarrierStubC2 {
+private:
+  const Address  _ref_addr;
+  const Register _ref;
+
+protected:
+  ZLoadBarrierStubC2(const MachNode* node, Address ref_addr, Register ref);
+
+public:
+  static ZLoadBarrierStubC2* create(const MachNode* node, Address ref_addr, Register ref);
+
+  Address ref_addr() const;
+  Register ref() const;
+  address slow_path() const;
+
+  virtual Register result() const;
+  virtual void emit_code(MacroAssembler& masm);
+};
+
+class ZStoreBarrierStubC2 : public ZBarrierStubC2 {
+private:
+  const Address  _ref_addr;
+  const Register _new_zaddress;
+  const Register _new_zpointer;
+  const bool     _is_native;
+  const bool     _is_atomic;
+
+protected:
+  ZStoreBarrierStubC2(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic);
+
+public:
+  static ZStoreBarrierStubC2* create(const MachNode* node, Address ref_addr, Register new_zaddress, Register new_zpointer, bool is_native, bool is_atomic);
+
+  Address ref_addr() const;
+  Register new_zaddress() const;
+  Register new_zpointer() const;
+  bool is_native() const;
+  bool is_atomic() const;
+
+  virtual Register result() const;
+  virtual void emit_code(MacroAssembler& masm);
 };
 
 class ZBarrierSetC2 : public BarrierSetC2 {
 private:
   void compute_liveness_at_stubs() const;
+  void analyze_dominating_barriers_impl(Node_List& accesses, Node_List& access_dominators) const;
   void analyze_dominating_barriers() const;
 
 protected:
+  virtual Node* store_at_resolved(C2Access& access, C2AccessValue& val) const;
   virtual Node* load_at_resolved(C2Access& access, const Type* val_type) const;
   virtual Node* atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access,
                                                Node* expected_val,
@@ -91,6 +140,8 @@ public:
   virtual void late_barrier_analysis() const;
   virtual int estimate_stub_size() const;
   virtual void emit_stubs(CodeBuffer& cb) const;
+  virtual void eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const;
+  virtual void eliminate_gc_barrier_data(Node* node) const;
 
 #ifndef PRODUCT
   virtual void dump_barrier_data(const MachNode* mach, outputStream* st) const;
diff --git a/src/hotspot/share/gc/z/shared/vmStructs_z_shared.hpp b/src/hotspot/share/gc/z/shared/vmStructs_z_shared.hpp
new file mode 100644
index 00000000000..f0c03abbf7a
--- /dev/null
+++ b/src/hotspot/share/gc/z/shared/vmStructs_z_shared.hpp
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_SHARED_VMSTRUCTS_Z_SHARED_HPP
+#define SHARE_GC_Z_SHARED_VMSTRUCTS_Z_SHARED_HPP
+
+#include "gc/x/vmStructs_x.hpp"
+#include "gc/z/vmStructs_z.hpp"
+
+#define VM_STRUCTS_Z_SHARED(nonstatic_field, volatile_nonstatic_field, static_field)    \
+  VM_STRUCTS_X(                                                                         \
+    nonstatic_field,                                                                    \
+    volatile_nonstatic_field,                                                           \
+    static_field)                                                                       \
+                                                                                        \
+  VM_STRUCTS_Z(                                                                         \
+    nonstatic_field,                                                                    \
+    volatile_nonstatic_field,                                                           \
+    static_field)
+
+#define VM_INT_CONSTANTS_Z_SHARED(declare_constant, declare_constant_with_value)        \
+  VM_INT_CONSTANTS_X(                                                                   \
+    declare_constant,                                                                   \
+    declare_constant_with_value)                                                        \
+                                                                                        \
+  VM_INT_CONSTANTS_Z(                                                                   \
+    declare_constant,                                                                   \
+    declare_constant_with_value)
+
+#define VM_LONG_CONSTANTS_Z_SHARED(declare_constant)                                    \
+  VM_LONG_CONSTANTS_X(                                                                  \
+    declare_constant)                                                                   \
+                                                                                        \
+  VM_LONG_CONSTANTS_Z(                                                                  \
+    declare_constant)
+
+#define VM_TYPES_Z_SHARED(declare_type, declare_toplevel_type, declare_integer_type)    \
+  VM_TYPES_X(                                                                           \
+    declare_type,                                                                       \
+    declare_toplevel_type,                                                              \
+    declare_integer_type)                                                               \
+                                                                                        \
+  VM_TYPES_Z(                                                                           \
+    declare_type,                                                                       \
+    declare_toplevel_type,                                                              \
+    declare_integer_type)
+
+#endif // SHARE_GC_Z_SHARED_VMSTRUCTS_Z_SHARED_HPP
diff --git a/src/hotspot/share/gc/z/shared/zSharedArguments.cpp b/src/hotspot/share/gc/z/shared/zSharedArguments.cpp
new file mode 100644
index 00000000000..8a00a851acb
--- /dev/null
+++ b/src/hotspot/share/gc/z/shared/zSharedArguments.cpp
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gcArguments.hpp"
+#include "gc/x/xArguments.hpp"
+#include "gc/z/shared/zSharedArguments.hpp"
+#include "gc/z/zArguments.hpp"
+#include "runtime/globals.hpp"
+#include "runtime/globals_extension.hpp"
+#include "runtime/java.hpp"
+
+void ZSharedArguments::initialize_alignments() {
+  if (ZGenerational) {
+    ZArguments::initialize_alignments();
+  } else {
+    XArguments::initialize_alignments();
+  }
+}
+
+void ZSharedArguments::initialize() {
+  GCArguments::initialize();
+
+  if (ZGenerational) {
+    ZArguments::initialize();
+  } else {
+    XArguments::initialize();
+  }
+}
+
+size_t ZSharedArguments::heap_virtual_to_physical_ratio() {
+  if (ZGenerational) {
+    return ZArguments::heap_virtual_to_physical_ratio();
+  } else {
+    return XArguments::heap_virtual_to_physical_ratio();
+  }
+}
+
+size_t ZSharedArguments::conservative_max_heap_alignment() {
+  return 0;
+}
+
+CollectedHeap* ZSharedArguments::create_heap() {
+  if (ZGenerational) {
+    return ZArguments::create_heap();
+  } else {
+    return XArguments::create_heap();
+  }
+}
+
+bool ZSharedArguments::is_supported() const {
+  if (ZGenerational) {
+    return ZArguments::is_os_supported();
+  } else {
+    return XArguments::is_os_supported();
+  }
+}
diff --git a/src/hotspot/share/gc/z/shared/zSharedArguments.hpp b/src/hotspot/share/gc/z/shared/zSharedArguments.hpp
new file mode 100644
index 00000000000..74659f581b9
--- /dev/null
+++ b/src/hotspot/share/gc/z/shared/zSharedArguments.hpp
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_SHARED_ZSHAREDARGUMENTS_HPP
+#define SHARE_GC_Z_SHARED_ZSHAREDARGUMENTS_HPP
+
+#include "gc/shared/gcArguments.hpp"
+
+class CollectedHeap;
+
+class ZSharedArguments : public GCArguments {
+private:
+  virtual void initialize_alignments();
+
+  virtual void initialize();
+  virtual size_t conservative_max_heap_alignment();
+  virtual size_t heap_virtual_to_physical_ratio();
+  virtual CollectedHeap* create_heap();
+
+  virtual bool is_supported() const;
+
+  bool is_os_supported() const;
+};
+
+#endif // SHARE_GC_Z_SHARED_ZSHAREDARGUMENTS_HPP
diff --git a/src/hotspot/share/gc/z/shared/z_shared_globals.hpp b/src/hotspot/share/gc/z/shared/z_shared_globals.hpp
new file mode 100644
index 00000000000..34e5e875d48
--- /dev/null
+++ b/src/hotspot/share/gc/z/shared/z_shared_globals.hpp
@@ -0,0 +1,103 @@
+/*
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_SHARED_Z_SHARED_GLOBALS_HPP
+#define SHARE_GC_Z_SHARED_Z_SHARED_GLOBALS_HPP
+
+#include "gc/x/x_globals.hpp"
+#include "gc/z/z_globals.hpp"
+
+#define GC_Z_SHARED_FLAGS(develop,                                          \
+                          develop_pd,                                       \
+                          product,                                          \
+                          product_pd,                                       \
+                          notproduct,                                       \
+                          range,                                            \
+                          constraint)                                       \
+                                                                            \
+  product(double, ZAllocationSpikeTolerance, 2.0,                           \
+          "Allocation spike tolerance factor")                              \
+                                                                            \
+  /* Updated in arguments parsing to ZGenerational ? 5.0 : 25.0 */          \
+  product(double, ZFragmentationLimit, 0 /* ignored */,                     \
+          "Maximum allowed heap fragmentation")                             \
+          range(0, 100)                                                     \
+                                                                            \
+  product(size_t, ZMarkStackSpaceLimit, 8*G,                                \
+          "Maximum number of bytes allocated for mark stacks")              \
+          range(32*M, 1024*G)                                               \
+                                                                            \
+  product(double, ZCollectionInterval, 0,                                   \
+          "Force GC at a fixed time interval (in seconds). "                \
+          "Backwards compatible alias for ZCollectionIntervalMajor")        \
+                                                                            \
+  product(bool, ZProactive, true,                                           \
+          "Enable proactive GC cycles")                                     \
+                                                                            \
+  product(bool, ZUncommit, true,                                            \
+          "Uncommit unused memory")                                         \
+                                                                            \
+  product(uintx, ZUncommitDelay, 5 * 60,                                    \
+          "Uncommit memory if it has been unused for the specified "        \
+          "amount of time (in seconds)")                                    \
+                                                                            \
+  product(uint, ZStatisticsInterval, 10, DIAGNOSTIC,                        \
+          "Time between statistics print outs (in seconds)")                \
+          range(1, (uint)-1)                                                \
+                                                                            \
+  product(bool, ZStressRelocateInPlace, false, DIAGNOSTIC,                  \
+          "Always relocate pages in-place")                                 \
+                                                                            \
+  product(bool, ZVerifyRoots, trueInDebug, DIAGNOSTIC,                      \
+          "Verify roots")                                                   \
+                                                                            \
+  product(bool, ZVerifyObjects, false, DIAGNOSTIC,                          \
+          "Verify objects")                                                 \
+                                                                            \
+  product(bool, ZVerifyMarking, trueInDebug, DIAGNOSTIC,                    \
+          "Verify marking stacks")                                          \
+                                                                            \
+  product(bool, ZVerifyForwarding, false, DIAGNOSTIC,                       \
+          "Verify forwarding tables")                                       \
+                                                                            \
+  GC_X_FLAGS(                                                               \
+    develop,                                                                \
+    develop_pd,                                                             \
+    product,                                                                \
+    product_pd,                                                             \
+    notproduct,                                                             \
+    range,                                                                  \
+    constraint)                                                             \
+                                                                            \
+  GC_Z_FLAGS(                                                               \
+    develop,                                                                \
+    develop_pd,                                                             \
+    product,                                                                \
+    product_pd,                                                             \
+    notproduct,                                                             \
+    range,                                                                  \
+    constraint)
+
+// end of GC_Z_SHARED_FLAGS
+
+#endif // SHARE_GC_Z_SHARED_Z_SHARED_GLOBALS_HPP
diff --git a/src/hotspot/share/gc/z/vmStructs_z.cpp b/src/hotspot/share/gc/z/vmStructs_z.cpp
index c86d11c8189..cb3948851d6 100644
--- a/src/hotspot/share/gc/z/vmStructs_z.cpp
+++ b/src/hotspot/share/gc/z/vmStructs_z.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,16 +23,17 @@
 
 #include "precompiled.hpp"
 #include "gc/z/vmStructs_z.hpp"
+#include "gc/z/zAddress.hpp"
 
 ZGlobalsForVMStructs::ZGlobalsForVMStructs() :
-    _ZGlobalPhase(&ZGlobalPhase),
-    _ZGlobalSeqNum(&ZGlobalSeqNum),
     _ZAddressOffsetMask(&ZAddressOffsetMask),
-    _ZAddressMetadataMask(&ZAddressMetadataMask),
-    _ZAddressMetadataFinalizable(&ZAddressMetadataFinalizable),
-    _ZAddressGoodMask(&ZAddressGoodMask),
-    _ZAddressBadMask(&ZAddressBadMask),
-    _ZAddressWeakBadMask(&ZAddressWeakBadMask),
+    _ZPointerLoadGoodMask(&ZPointerLoadGoodMask),
+    _ZPointerLoadBadMask(&ZPointerLoadBadMask),
+    _ZPointerLoadShift(const_cast<size_t*>(&ZPointerLoadShift)),
+    _ZPointerMarkGoodMask(&ZPointerMarkGoodMask),
+    _ZPointerMarkBadMask(&ZPointerMarkBadMask),
+    _ZPointerStoreGoodMask(&ZPointerStoreGoodMask),
+    _ZPointerStoreBadMask(&ZPointerStoreBadMask),
     _ZObjectAlignmentSmallShift(&ZObjectAlignmentSmallShift),
     _ZObjectAlignmentSmall(&ZObjectAlignmentSmall) {
 }
diff --git a/src/hotspot/share/gc/z/vmStructs_z.hpp b/src/hotspot/share/gc/z/vmStructs_z.hpp
index 3c0eb9f74ea..47fa6ac3021 100644
--- a/src/hotspot/share/gc/z/vmStructs_z.hpp
+++ b/src/hotspot/share/gc/z/vmStructs_z.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,6 +30,7 @@
 #include "gc/z/zGranuleMap.hpp"
 #include "gc/z/zHeap.hpp"
 #include "gc/z/zPageAllocator.hpp"
+#include "gc/z/zPageType.hpp"
 #include "utilities/macros.hpp"
 
 // Expose some ZGC globals to the SA agent.
@@ -41,16 +42,17 @@ public:
 
   ZGlobalsForVMStructs();
 
-  uint32_t* _ZGlobalPhase;
-
-  uint32_t* _ZGlobalSeqNum;
-
   uintptr_t* _ZAddressOffsetMask;
-  uintptr_t* _ZAddressMetadataMask;
-  uintptr_t* _ZAddressMetadataFinalizable;
-  uintptr_t* _ZAddressGoodMask;
-  uintptr_t* _ZAddressBadMask;
-  uintptr_t* _ZAddressWeakBadMask;
+
+  uintptr_t* _ZPointerLoadGoodMask;
+  uintptr_t* _ZPointerLoadBadMask;
+  size_t*    _ZPointerLoadShift;
+
+  uintptr_t* _ZPointerMarkGoodMask;
+  uintptr_t* _ZPointerMarkBadMask;
+
+  uintptr_t* _ZPointerStoreGoodMask;
+  uintptr_t* _ZPointerStoreBadMask;
 
   const int* _ZObjectAlignmentSmallShift;
   const int* _ZObjectAlignmentSmall;
@@ -60,30 +62,29 @@ typedef ZGranuleMap<ZPage*> ZGranuleMapForPageTable;
 typedef ZGranuleMap<ZForwarding*> ZGranuleMapForForwarding;
 typedef ZAttachedArray<ZForwarding, ZForwardingEntry> ZAttachedArrayForForwarding;
 
-#define VM_STRUCTS_ZGC(nonstatic_field, volatile_nonstatic_field, static_field)                      \
+#define VM_STRUCTS_Z(nonstatic_field, volatile_nonstatic_field, static_field)                        \
   static_field(ZGlobalsForVMStructs,            _instance_p,          ZGlobalsForVMStructs*)         \
-  nonstatic_field(ZGlobalsForVMStructs,         _ZGlobalPhase,        uint32_t*)                     \
-  nonstatic_field(ZGlobalsForVMStructs,         _ZGlobalSeqNum,       uint32_t*)                     \
-  nonstatic_field(ZGlobalsForVMStructs,         _ZAddressOffsetMask,  uintptr_t*)                    \
-  nonstatic_field(ZGlobalsForVMStructs,         _ZAddressMetadataMask, uintptr_t*)                   \
-  nonstatic_field(ZGlobalsForVMStructs,         _ZAddressMetadataFinalizable, uintptr_t*)            \
-  nonstatic_field(ZGlobalsForVMStructs,         _ZAddressGoodMask,    uintptr_t*)                    \
-  nonstatic_field(ZGlobalsForVMStructs,         _ZAddressBadMask,     uintptr_t*)                    \
-  nonstatic_field(ZGlobalsForVMStructs,         _ZAddressWeakBadMask, uintptr_t*)                    \
+                                                                                                     \
+  nonstatic_field(ZGlobalsForVMStructs,         _ZAddressOffsetMask,         uintptr_t*)             \
+  nonstatic_field(ZGlobalsForVMStructs,         _ZPointerLoadGoodMask,       uintptr_t*)             \
+  nonstatic_field(ZGlobalsForVMStructs,         _ZPointerLoadBadMask,        uintptr_t*)             \
+  nonstatic_field(ZGlobalsForVMStructs,         _ZPointerLoadShift,          size_t*)                \
+  nonstatic_field(ZGlobalsForVMStructs,         _ZPointerMarkGoodMask,       uintptr_t*)             \
+  nonstatic_field(ZGlobalsForVMStructs,         _ZPointerMarkBadMask,        uintptr_t*)             \
+  nonstatic_field(ZGlobalsForVMStructs,         _ZPointerStoreGoodMask,      uintptr_t*)             \
+  nonstatic_field(ZGlobalsForVMStructs,         _ZPointerStoreBadMask,       uintptr_t*)             \
   nonstatic_field(ZGlobalsForVMStructs,         _ZObjectAlignmentSmallShift, const int*)             \
-  nonstatic_field(ZGlobalsForVMStructs,         _ZObjectAlignmentSmall, const int*)                  \
+  nonstatic_field(ZGlobalsForVMStructs,         _ZObjectAlignmentSmall,      const int*)             \
                                                                                                      \
   nonstatic_field(ZCollectedHeap,               _heap,                ZHeap)                         \
                                                                                                      \
   nonstatic_field(ZHeap,                        _page_allocator,      ZPageAllocator)                \
   nonstatic_field(ZHeap,                        _page_table,          ZPageTable)                    \
-  nonstatic_field(ZHeap,                        _forwarding_table,    ZForwardingTable)              \
-  nonstatic_field(ZHeap,                        _relocate,            ZRelocate)                     \
                                                                                                      \
-  nonstatic_field(ZPage,                        _type,                const uint8_t)                 \
-  nonstatic_field(ZPage,                        _seqnum,              uint32_t)                      \
+  nonstatic_field(ZPage,                        _type,                const ZPageType)               \
+  volatile_nonstatic_field(ZPage,               _seqnum,              uint32_t)                      \
   nonstatic_field(ZPage,                        _virtual,             const ZVirtualMemory)          \
-  volatile_nonstatic_field(ZPage,               _top,                 uintptr_t)                     \
+  volatile_nonstatic_field(ZPage,               _top,                 zoffset_end)                   \
                                                                                                      \
   nonstatic_field(ZPageAllocator,               _max_capacity,        const size_t)                  \
   volatile_nonstatic_field(ZPageAllocator,      _capacity,            size_t)                        \
@@ -96,8 +97,8 @@ typedef ZAttachedArray<ZForwarding, ZForwardingEntry> ZAttachedArrayForForwardin
                                                                                                      \
   nonstatic_field(ZForwardingTable,             _map,                 ZGranuleMapForForwarding)      \
                                                                                                      \
-  nonstatic_field(ZVirtualMemory,               _start,               const uintptr_t)               \
-  nonstatic_field(ZVirtualMemory,               _end,                 const uintptr_t)               \
+  nonstatic_field(ZVirtualMemory,               _start,               const zoffset)                 \
+  nonstatic_field(ZVirtualMemory,               _end,                 const zoffset_end)             \
                                                                                                      \
   nonstatic_field(ZForwarding,                  _virtual,             const ZVirtualMemory)          \
   nonstatic_field(ZForwarding,                  _object_alignment_shift, const size_t)               \
@@ -106,15 +107,14 @@ typedef ZAttachedArray<ZForwarding, ZForwardingEntry> ZAttachedArrayForForwardin
   nonstatic_field(ZForwardingEntry,             _entry,               uint64_t)                      \
   nonstatic_field(ZAttachedArrayForForwarding,  _length,              const size_t)
 
-#define VM_INT_CONSTANTS_ZGC(declare_constant, declare_constant_with_value)                          \
-  declare_constant(ZPhaseRelocate)                                                                   \
-  declare_constant(ZPageTypeSmall)                                                                   \
-  declare_constant(ZPageTypeMedium)                                                                  \
-  declare_constant(ZPageTypeLarge)                                                                   \
+#define VM_INT_CONSTANTS_Z(declare_constant, declare_constant_with_value)                            \
+  declare_constant(ZPageType::small)                                                                 \
+  declare_constant(ZPageType::medium)                                                                \
+  declare_constant(ZPageType::large)                                                                 \
   declare_constant(ZObjectAlignmentMediumShift)                                                      \
   declare_constant(ZObjectAlignmentLargeShift)
 
-#define VM_LONG_CONSTANTS_ZGC(declare_constant)                                                      \
+#define VM_LONG_CONSTANTS_Z(declare_constant)                                                        \
   declare_constant(ZGranuleSizeShift)                                                                \
   declare_constant(ZPageSizeSmallShift)                                                              \
   declare_constant(ZPageSizeMediumShift)                                                             \
@@ -123,12 +123,15 @@ typedef ZAttachedArray<ZForwarding, ZForwardingEntry> ZAttachedArrayForForwardin
   declare_constant(ZAddressOffsetMask)                                                               \
   declare_constant(ZAddressOffsetMax)
 
-#define VM_TYPES_ZGC(declare_type, declare_toplevel_type, declare_integer_type)                      \
+#define VM_TYPES_Z(declare_type, declare_toplevel_type, declare_integer_type)                        \
+  declare_toplevel_type(zoffset)                                                                     \
+  declare_toplevel_type(zoffset_end)                                                                 \
   declare_toplevel_type(ZGlobalsForVMStructs)                                                        \
   declare_type(ZCollectedHeap, CollectedHeap)                                                        \
   declare_toplevel_type(ZHeap)                                                                       \
   declare_toplevel_type(ZRelocate)                                                                   \
   declare_toplevel_type(ZPage)                                                                       \
+  declare_toplevel_type(ZPageType)                                                                   \
   declare_toplevel_type(ZPageAllocator)                                                              \
   declare_toplevel_type(ZPageTable)                                                                  \
   declare_toplevel_type(ZAttachedArrayForForwarding)                                                 \
diff --git a/src/hotspot/share/gc/z/zAbort.cpp b/src/hotspot/share/gc/z/zAbort.cpp
index 1ac18ce9970..938e77b7d80 100644
--- a/src/hotspot/share/gc/z/zAbort.cpp
+++ b/src/hotspot/share/gc/z/zAbort.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,5 +28,5 @@
 volatile bool ZAbort::_should_abort = false;
 
 void ZAbort::abort() {
-  Atomic::release_store_fence(&_should_abort, true);
+  Atomic::store(&_should_abort, true);
 }
diff --git a/src/hotspot/share/gc/z/zAbort.hpp b/src/hotspot/share/gc/z/zAbort.hpp
index f87bca8c0a3..925b0a79ac3 100644
--- a/src/hotspot/share/gc/z/zAbort.hpp
+++ b/src/hotspot/share/gc/z/zAbort.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -35,4 +35,12 @@ public:
   static void abort();
 };
 
+// Macro to execute a abortion check
+#define abortpoint()               \
+  do {                             \
+    if (ZAbort::should_abort()) {  \
+      return;                      \
+    }                              \
+  } while (false)
+
 #endif // SHARE_GC_Z_ZABORT_HPP
diff --git a/src/hotspot/share/gc/z/zAbort.inline.hpp b/src/hotspot/share/gc/z/zAbort.inline.hpp
index a8af1b7f720..0037f7ec448 100644
--- a/src/hotspot/share/gc/z/zAbort.inline.hpp
+++ b/src/hotspot/share/gc/z/zAbort.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,7 +29,7 @@
 #include "runtime/atomic.hpp"
 
 inline bool ZAbort::should_abort() {
-  return Atomic::load_acquire(&_should_abort);
+  return Atomic::load(&_should_abort);
 }
 
 #endif // SHARE_GC_Z_ZABORT_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zAddress.cpp b/src/hotspot/share/gc/z/zAddress.cpp
index cfa7c04d3d2..d1c199fad07 100644
--- a/src/hotspot/share/gc/z/zAddress.cpp
+++ b/src/hotspot/share/gc/z/zAddress.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,37 +22,129 @@
  */
 
 #include "precompiled.hpp"
-#include "gc/z/zAddress.hpp"
-#include "gc/z/zGlobals.hpp"
+#include "gc/shared/barrierSet.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zVerify.hpp"
+#include "oops/oopsHierarchy.hpp"
+#include "runtime/java.hpp"
+#include "utilities/formatBuffer.hpp"
 
-void ZAddress::set_good_mask(uintptr_t mask) {
-  ZAddressGoodMask = mask;
-  ZAddressBadMask = ZAddressGoodMask ^ ZAddressMetadataMask;
-  ZAddressWeakBadMask = (ZAddressGoodMask | ZAddressMetadataRemapped | ZAddressMetadataFinalizable) ^ ZAddressMetadataMask;
+size_t     ZAddressHeapBaseShift;
+size_t     ZAddressHeapBase;
+
+size_t     ZAddressOffsetBits;
+uintptr_t  ZAddressOffsetMask;
+size_t     ZAddressOffsetMax;
+
+uintptr_t  ZPointerRemapped;
+uintptr_t  ZPointerRemappedYoungMask;
+uintptr_t  ZPointerRemappedOldMask;
+uintptr_t  ZPointerMarkedYoung;
+uintptr_t  ZPointerMarkedOld;
+uintptr_t  ZPointerFinalizable;
+uintptr_t  ZPointerRemembered;
+
+uintptr_t  ZPointerLoadGoodMask;
+uintptr_t  ZPointerLoadBadMask;
+
+uintptr_t  ZPointerMarkGoodMask;
+uintptr_t  ZPointerMarkBadMask;
+
+uintptr_t  ZPointerStoreGoodMask;
+uintptr_t  ZPointerStoreBadMask;
+
+uintptr_t  ZPointerVectorLoadBadMask[8];
+uintptr_t  ZPointerVectorStoreBadMask[8];
+uintptr_t  ZPointerVectorStoreGoodMask[8];
+
+static uint32_t* ZPointerCalculateStoreGoodMaskLowOrderBitsAddr() {
+  const uintptr_t addr = reinterpret_cast<uintptr_t>(&ZPointerStoreGoodMask);
+  return reinterpret_cast<uint32_t*>(addr + ZPointerStoreGoodMaskLowOrderBitsOffset);
 }
 
-void ZAddress::initialize() {
+uint32_t*  ZPointerStoreGoodMaskLowOrderBitsAddr = ZPointerCalculateStoreGoodMaskLowOrderBitsAddr();
+
+static void set_vector_mask(uintptr_t vector_mask[], uintptr_t mask) {
+  for (int i = 0; i < 8; ++i) {
+    vector_mask[i] = mask;
+  }
+}
+
+void ZGlobalsPointers::set_good_masks() {
+  ZPointerRemapped = ZPointerRemappedOldMask & ZPointerRemappedYoungMask;
+
+  ZPointerLoadGoodMask  = ZPointer::remap_bits(ZPointerRemapped);
+  ZPointerMarkGoodMask  = ZPointerLoadGoodMask | ZPointerMarkedYoung | ZPointerMarkedOld;
+  ZPointerStoreGoodMask = ZPointerMarkGoodMask | ZPointerRemembered;
+
+  ZPointerLoadBadMask  = ZPointerLoadGoodMask  ^ ZPointerLoadMetadataMask;
+  ZPointerMarkBadMask  = ZPointerMarkGoodMask  ^ ZPointerMarkMetadataMask;
+  ZPointerStoreBadMask = ZPointerStoreGoodMask ^ ZPointerStoreMetadataMask;
+
+  set_vector_mask(ZPointerVectorLoadBadMask, ZPointerLoadBadMask);
+  set_vector_mask(ZPointerVectorStoreBadMask, ZPointerStoreBadMask);
+  set_vector_mask(ZPointerVectorStoreGoodMask, ZPointerStoreGoodMask);
+
+  pd_set_good_masks();
+}
+
+static void initialize_check_oop_function() {
+#ifdef CHECK_UNHANDLED_OOPS
+  if (ZVerifyOops) {
+    // Enable extra verification of usages of oops in oopsHierarchy.hpp
+    check_oop_function = [](oopDesc* obj) {
+      (void)to_zaddress(obj);
+    };
+  }
+#endif
+}
+
+void ZGlobalsPointers::initialize() {
   ZAddressOffsetBits = ZPlatformAddressOffsetBits();
   ZAddressOffsetMask = (((uintptr_t)1 << ZAddressOffsetBits) - 1) << ZAddressOffsetShift;
   ZAddressOffsetMax = (uintptr_t)1 << ZAddressOffsetBits;
 
-  ZAddressMetadataShift = ZPlatformAddressMetadataShift();
-  ZAddressMetadataMask = (((uintptr_t)1 << ZAddressMetadataBits) - 1) << ZAddressMetadataShift;
+  // Check max supported heap size
+  if (MaxHeapSize > ZAddressOffsetMax) {
+    vm_exit_during_initialization(
+        err_msg("Java heap too large (max supported heap size is " SIZE_FORMAT "G)",
+                ZAddressOffsetMax / G));
+  }
 
-  ZAddressMetadataMarked0 = (uintptr_t)1 << (ZAddressMetadataShift + 0);
-  ZAddressMetadataMarked1 = (uintptr_t)1 << (ZAddressMetadataShift + 1);
-  ZAddressMetadataRemapped = (uintptr_t)1 << (ZAddressMetadataShift + 2);
-  ZAddressMetadataFinalizable = (uintptr_t)1 << (ZAddressMetadataShift + 3);
+  ZAddressHeapBaseShift = ZPlatformAddressHeapBaseShift();
+  ZAddressHeapBase = (uintptr_t)1 << ZAddressHeapBaseShift;
 
-  ZAddressMetadataMarked = ZAddressMetadataMarked0;
-  set_good_mask(ZAddressMetadataRemapped);
+  ZPointerRemappedYoungMask = ZPointerRemapped10 | ZPointerRemapped00;
+  ZPointerRemappedOldMask = ZPointerRemapped01 | ZPointerRemapped00;
+  ZPointerMarkedYoung = ZPointerMarkedYoung0;
+  ZPointerMarkedOld = ZPointerMarkedOld0;
+  ZPointerFinalizable = ZPointerFinalizable0;
+  ZPointerRemembered = ZPointerRemembered0;
+
+  set_good_masks();
+
+  initialize_check_oop_function();
 }
 
-void ZAddress::flip_to_marked() {
-  ZAddressMetadataMarked ^= (ZAddressMetadataMarked0 | ZAddressMetadataMarked1);
-  set_good_mask(ZAddressMetadataMarked);
+void ZGlobalsPointers::flip_young_mark_start() {
+  ZPointerMarkedYoung ^= (ZPointerMarkedYoung0 | ZPointerMarkedYoung1);
+  ZPointerRemembered ^= (ZPointerRemembered0 | ZPointerRemembered1);
+  set_good_masks();
 }
 
-void ZAddress::flip_to_remapped() {
-  set_good_mask(ZAddressMetadataRemapped);
+void ZGlobalsPointers::flip_young_relocate_start() {
+  ZPointerRemappedYoungMask ^= ZPointerRemappedMask;
+  set_good_masks();
+}
+
+void ZGlobalsPointers::flip_old_mark_start() {
+  ZPointerMarkedOld ^= (ZPointerMarkedOld0 | ZPointerMarkedOld1);
+  ZPointerFinalizable ^= (ZPointerFinalizable0 | ZPointerFinalizable1);
+  set_good_masks();
+}
+
+void ZGlobalsPointers::flip_old_relocate_start() {
+  ZPointerRemappedOldMask ^= ZPointerRemappedMask;
+  set_good_masks();
 }
diff --git a/src/hotspot/share/gc/z/zAddress.hpp b/src/hotspot/share/gc/z/zAddress.hpp
index 2908c37bbe6..86b9550c043 100644
--- a/src/hotspot/share/gc/z/zAddress.hpp
+++ b/src/hotspot/share/gc/z/zAddress.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,42 +26,288 @@
 
 #include "memory/allStatic.hpp"
 #include "utilities/globalDefinitions.hpp"
+#include CPU_HEADER(gc/z/zAddress)
+
+// One bit that denotes where the heap start. All uncolored
+// oops have this bit set, plus an offset within the heap.
+extern uintptr_t  ZAddressHeapBase;
+extern uintptr_t  ZAddressHeapBaseShift;
+
+// Describes the maximal offset inside the heap.
+extern size_t    ZAddressOffsetBits;
+const  size_t    ZAddressOffsetShift = 0;
+extern uintptr_t ZAddressOffsetMask;
+extern size_t    ZAddressOffsetMax;
+
+// Layout of metadata bits in colored pointer / zpointer.
+//
+// A zpointer is a combination of the address bits (heap base bit + offset)
+// and two low-order metadata bytes, with the following layout:
+//
+// RRRRMMmmFFrr0000
+// ****               : Used by load barrier
+// **********         : Used by mark barrier
+// ************       : Used by store barrier
+//             ****   : Reserved bits
+//
+// The table below describes what each color does.
+//
+// +-------------+-------------------+--------------------------+
+// | Bit pattern | Description       | Included colors          |
+// +-------------+-------------------+--------------------------+
+// |     rr      | Remembered bits   | Remembered[0, 1]         |
+// +-------------+-------------------+--------------------------+
+// |     FF      | Finalizable bits  | Finalizable[0, 1]        |
+// +-------------+-------------------+--------------------------+
+// |     mm      | Marked young bits | MarkedYoung[0, 1]        |
+// +-------------+-------------------+--------------------------+
+// |     MM      | Marked old bits   | MarkedOld[0, 1]          |
+// +-------------+-------------------+--------------------------+
+// |    RRRR     | Remapped bits     | Remapped[00, 01, 10, 11] |
+// +-------------+-------------------+--------------------------+
+//
+// The low order zero address bits sometimes overlap with the high order zero metadata
+// bits, depending on the remapped bit being set.
+//
+//             vvv- overlapping address and metadata zeros
+//    aaa...aaa0001MMmmFFrr0000 = Remapped00 zpointer
+//
+//             vv-- overlapping address and metadata zeros
+//   aaa...aaa00010MMmmFFrr0000 = Remapped01 zpointer
+//
+//             v--- overlapping address and metadata zero
+//  aaa...aaa000100MMmmFFrr0000 = Remapped10 zpointer
+//
+//             ---- no overlapping address and metadata zeros
+// aaa...aaa0001000MMmmFFrr0000 = Remapped11 zpointer
+//
+// The overlapping is performed because the x86 JIT-compiled load barriers expect the
+// address bits to start right after the load-good bit. It allows combining the good
+// bit check and unmasking into a single speculative shift instruction. On AArch64 we
+// don't do this, and hence there are no overlapping address and  metadata zeros there.
+//
+// The remapped bits are notably not grouped into two sets of bits, one for the young
+// collection and one for the old collection, like the other bits. The reason is that
+// the load barrier is only compatible with bit patterns where there is a single zero in
+// its bits of operation (the load metadata bit mask). Instead, the single bit that we
+// set encodes the combined state of a conceptual RemappedYoung[0, 1] and
+// RemappedOld[0, 1] pair. The encoding scheme is that the shift of the load good bit,
+// minus the shift of the load metadata bit start encodes the numbers 0, 1, 2 and 3.
+// These numbers in binary correspond to 00, 01, 10 and 11. The low order bit in said
+// numbers correspond to the simulated RemappedYoung[0, 1] value, and the high order bit
+// corresponds to the simulated RemappedOld[0, 1] value. On AArch64, the remap bits
+// of zpointers are the complement of this bit. So there are 3 good bits and one bad bit
+// instead. This lends itself better to AArch64 instructions.
+//
+// We decide the bit to be taken by having the RemappedYoungMask and RemappedOldMask
+// variables, which alternate between what two bits they accept for their corresponding
+// old and young phase. The Remapped bit is chosen by taking the intersection of those
+// two variables.
+//
+// RemappedOldMask alternates between these two bit patterns:
+//
+//  RemappedOld0 => 0011
+//  RemappedOld1 => 1100
+//
+// RemappedYoungMask alternates between these two bit patterns:
+//
+//  RemappedYoung0 => 0101
+//  RemappedYoung1 => 1010
+//
+// The corresponding intersections look like this:
+//
+//  RemappedOld0 & RemappedYoung0 = 0001 = Remapped00
+//  RemappedOld0 & RemappedYoung1 = 0010 = Remapped01
+//  RemappedOld1 & RemappedYoung0 = 0100 = Remapped10
+//  RemappedOld1 & RemappedYoung1 = 1000 = Remapped11
+
+constexpr uintptr_t z_pointer_mask(size_t shift, size_t bits) {
+  return (((uintptr_t)1 << bits) - 1) << shift;
+}
+
+constexpr uintptr_t z_pointer_bit(size_t shift, size_t offset) {
+  return (uintptr_t)1 << (shift + offset);
+}
+
+// Reserved bits
+const size_t      ZPointerReservedShift   = 0;
+const size_t      ZPointerReservedBits    = 4;
+const uintptr_t   ZPointerReservedMask    = z_pointer_mask(ZPointerReservedShift, ZPointerReservedBits);
+
+const uintptr_t   ZPointerReserved0       = z_pointer_bit(ZPointerReservedShift, 0);
+const uintptr_t   ZPointerReserved1       = z_pointer_bit(ZPointerReservedShift, 1);
+const uintptr_t   ZPointerReserved2       = z_pointer_bit(ZPointerReservedShift, 2);
+const uintptr_t   ZPointerReserved3       = z_pointer_bit(ZPointerReservedShift, 3);
+
+// Remembered set bits
+const size_t      ZPointerRememberedShift = ZPointerReservedShift + ZPointerReservedBits;
+const size_t      ZPointerRememberedBits  = 2;
+const uintptr_t   ZPointerRememberedMask  = z_pointer_mask(ZPointerRememberedShift, ZPointerRememberedBits);
+
+const uintptr_t   ZPointerRemembered0     = z_pointer_bit(ZPointerRememberedShift, 0);
+const uintptr_t   ZPointerRemembered1     = z_pointer_bit(ZPointerRememberedShift, 1);
+
+// Marked bits
+const size_t      ZPointerMarkedShift     = ZPointerRememberedShift + ZPointerRememberedBits;
+const size_t      ZPointerMarkedBits      = 6;
+const uintptr_t   ZPointerMarkedMask      = z_pointer_mask(ZPointerMarkedShift, ZPointerMarkedBits);
+
+const uintptr_t   ZPointerFinalizable0    = z_pointer_bit(ZPointerMarkedShift, 0);
+const uintptr_t   ZPointerFinalizable1    = z_pointer_bit(ZPointerMarkedShift, 1);
+const uintptr_t   ZPointerMarkedYoung0    = z_pointer_bit(ZPointerMarkedShift, 2);
+const uintptr_t   ZPointerMarkedYoung1    = z_pointer_bit(ZPointerMarkedShift, 3);
+const uintptr_t   ZPointerMarkedOld0      = z_pointer_bit(ZPointerMarkedShift, 4);
+const uintptr_t   ZPointerMarkedOld1      = z_pointer_bit(ZPointerMarkedShift, 5);
+
+// Remapped bits
+const size_t      ZPointerRemappedShift   = ZPointerMarkedShift + ZPointerMarkedBits;
+const size_t      ZPointerRemappedBits    = 4;
+const uintptr_t   ZPointerRemappedMask    = z_pointer_mask(ZPointerRemappedShift, ZPointerRemappedBits);
+
+const uintptr_t   ZPointerRemapped00      = z_pointer_bit(ZPointerRemappedShift, 0);
+const uintptr_t   ZPointerRemapped01      = z_pointer_bit(ZPointerRemappedShift, 1);
+const uintptr_t   ZPointerRemapped10      = z_pointer_bit(ZPointerRemappedShift, 2);
+const uintptr_t   ZPointerRemapped11      = z_pointer_bit(ZPointerRemappedShift, 3);
+
+// The shift table is tightly coupled with the zpointer layout given above
+constexpr int     ZPointerLoadShiftTable[] = {
+  ZPointerRemappedShift + ZPointerRemappedShift, // [0] Null
+  ZPointerRemappedShift + 1,                     // [1] Remapped00
+  ZPointerRemappedShift + 2,                     // [2] Remapped01
+  0,
+  ZPointerRemappedShift + 3,                     // [4] Remapped10
+  0,
+  0,
+  0,
+  ZPointerRemappedShift + 4                      // [8] Remapped11
+};
+
+// Barrier metadata masks
+const uintptr_t   ZPointerLoadMetadataMask  = ZPointerRemappedMask;
+const uintptr_t   ZPointerMarkMetadataMask  = ZPointerLoadMetadataMask | ZPointerMarkedMask;
+const uintptr_t   ZPointerStoreMetadataMask = ZPointerMarkMetadataMask | ZPointerRememberedMask;
+const uintptr_t   ZPointerAllMetadataMask   = ZPointerStoreMetadataMask;
+
+// The current expected bit
+extern uintptr_t  ZPointerRemapped;
+extern uintptr_t  ZPointerMarkedOld;
+extern uintptr_t  ZPointerMarkedYoung;
+extern uintptr_t  ZPointerFinalizable;
+extern uintptr_t  ZPointerRemembered;
+
+// The current expected remap bit for the young (or old) collection is either of two bits.
+// The other collection alternates the bits, so we need to use a mask.
+extern uintptr_t  ZPointerRemappedYoungMask;
+extern uintptr_t  ZPointerRemappedOldMask;
+
+// Good/bad masks
+extern uintptr_t  ZPointerLoadGoodMask;
+extern uintptr_t  ZPointerLoadBadMask;
+
+extern uintptr_t  ZPointerMarkGoodMask;
+extern uintptr_t  ZPointerMarkBadMask;
+
+extern uintptr_t  ZPointerStoreGoodMask;
+extern uintptr_t  ZPointerStoreBadMask;
+
+extern uintptr_t  ZPointerVectorLoadBadMask[8];
+extern uintptr_t  ZPointerVectorStoreBadMask[8];
+extern uintptr_t  ZPointerVectorStoreGoodMask[8];
+
+// The bad mask is 64 bit. Its low order 32 bits contain all possible value combinations
+// that this mask will have. Therefore, the memory where the 32 low order bits are stored
+// can be used as a 32 bit GC epoch counter, that has a different bit pattern every time
+// the bad mask is flipped. This provides a pointer to such 32 bits.
+extern uint32_t*  ZPointerStoreGoodMaskLowOrderBitsAddr;
+const int         ZPointerStoreGoodMaskLowOrderBitsOffset = LITTLE_ENDIAN_ONLY(0) BIG_ENDIAN_ONLY(4);
+
+// Offsets
+// - Virtual address range offsets
+// - Physical memory offsets
+enum class zoffset         : uintptr_t {};
+// Offsets including end of offset range
+enum class zoffset_end     : uintptr_t {};
+
+// Colored oop
+enum class zpointer        : uintptr_t { null = 0 };
+
+// Uncolored oop - safe to dereference
+enum class zaddress        : uintptr_t { null = 0 };
+
+// Uncolored oop - not safe to dereference, could point uncommitted memory
+enum class zaddress_unsafe : uintptr_t { null = 0 };
+
+class ZOffset : public AllStatic {
+public:
+  static zaddress address(zoffset offset);
+  static zaddress_unsafe address_unsafe(zoffset offset);
+};
+
+class ZPointer : public AllStatic {
+public:
+  static zaddress uncolor(zpointer ptr);
+  static zaddress uncolor_store_good(zpointer ptr);
+  static zaddress_unsafe uncolor_unsafe(zpointer ptr);
+  static zpointer set_remset_bits(zpointer ptr);
+
+  static bool is_load_bad(zpointer ptr);
+  static bool is_load_good(zpointer ptr);
+  static bool is_load_good_or_null(zpointer ptr);
+
+  static bool is_old_load_good(zpointer ptr);
+  static bool is_young_load_good(zpointer ptr);
+
+  static bool is_mark_bad(zpointer ptr);
+  static bool is_mark_good(zpointer ptr);
+  static bool is_mark_good_or_null(zpointer ptr);
+
+  static bool is_store_bad(zpointer ptr);
+  static bool is_store_good(zpointer ptr);
+  static bool is_store_good_or_null(zpointer ptr);
+
+  static bool is_marked_finalizable(zpointer ptr);
+  static bool is_marked_old(zpointer ptr);
+  static bool is_marked_young(zpointer ptr);
+  static bool is_marked_any_old(zpointer ptr);
+  static bool is_remapped(zpointer ptr);
+  static bool is_remembered_exact(zpointer ptr);
+
+  static constexpr int load_shift_lookup_index(uintptr_t value);
+  static constexpr int load_shift_lookup(uintptr_t value);
+  static uintptr_t remap_bits(uintptr_t colored);
+};
 
 class ZAddress : public AllStatic {
+public:
+  static zpointer color(zaddress addr, uintptr_t color);
+  static zpointer color(zaddress_unsafe addr, uintptr_t color);
+
+  static zoffset offset(zaddress addr);
+  static zoffset offset(zaddress_unsafe addr);
+
+  static zpointer load_good(zaddress addr, zpointer prev);
+  static zpointer finalizable_good(zaddress addr, zpointer prev);
+  static zpointer mark_good(zaddress addr, zpointer prev);
+  static zpointer mark_old_good(zaddress addr, zpointer prev);
+  static zpointer mark_young_good(zaddress addr, zpointer prev);
+  static zpointer store_good(zaddress addr);
+  static zpointer store_good_or_null(zaddress addr);
+};
+
+class ZGlobalsPointers : public AllStatic {
   friend class ZAddressTest;
 
 private:
-  static void set_good_mask(uintptr_t mask);
+  static void set_good_masks();
+  static void pd_set_good_masks();
 
 public:
   static void initialize();
 
-  static void flip_to_marked();
-  static void flip_to_remapped();
-
-  static bool is_null(uintptr_t value);
-  static bool is_bad(uintptr_t value);
-  static bool is_good(uintptr_t value);
-  static bool is_good_or_null(uintptr_t value);
-  static bool is_weak_bad(uintptr_t value);
-  static bool is_weak_good(uintptr_t value);
-  static bool is_weak_good_or_null(uintptr_t value);
-  static bool is_marked(uintptr_t value);
-  static bool is_marked_or_null(uintptr_t value);
-  static bool is_finalizable(uintptr_t value);
-  static bool is_finalizable_good(uintptr_t value);
-  static bool is_remapped(uintptr_t value);
-  static bool is_in(uintptr_t value);
-
-  static uintptr_t offset(uintptr_t value);
-  static uintptr_t good(uintptr_t value);
-  static uintptr_t good_or_null(uintptr_t value);
-  static uintptr_t finalizable_good(uintptr_t value);
-  static uintptr_t marked(uintptr_t value);
-  static uintptr_t marked0(uintptr_t value);
-  static uintptr_t marked1(uintptr_t value);
-  static uintptr_t remapped(uintptr_t value);
-  static uintptr_t remapped_or_null(uintptr_t value);
+  static void flip_young_mark_start();
+  static void flip_young_relocate_start();
+  static void flip_old_mark_start();
+  static void flip_old_relocate_start();
 };
 
 #endif // SHARE_GC_Z_ZADDRESS_HPP
diff --git a/src/hotspot/share/gc/z/zAddress.inline.hpp b/src/hotspot/share/gc/z/zAddress.inline.hpp
index a151e7182e4..7088a71ef7b 100644
--- a/src/hotspot/share/gc/z/zAddress.inline.hpp
+++ b/src/hotspot/share/gc/z/zAddress.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,24 +26,450 @@
 
 #include "gc/z/zAddress.hpp"
 
-#include "gc/z/zGlobals.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "oops/oop.hpp"
+#include "oops/oopsHierarchy.hpp"
+#include "runtime/atomic.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/macros.hpp"
 #include "utilities/powerOfTwo.hpp"
+#include CPU_HEADER_INLINE(gc/z/zAddress)
 
-inline bool ZAddress::is_null(uintptr_t value) {
-  return value == 0;
+// zoffset functions
+
+inline uintptr_t untype(zoffset offset) {
+  const uintptr_t value = static_cast<uintptr_t>(offset);
+  assert(value < ZAddressOffsetMax, "must have no other bits");
+  return value;
 }
 
-inline bool ZAddress::is_bad(uintptr_t value) {
-  return value & ZAddressBadMask;
+inline uintptr_t untype(zoffset_end offset) {
+  const uintptr_t value = static_cast<uintptr_t>(offset);
+  assert(value <= ZAddressOffsetMax, "must have no other bits");
+  return value;
 }
 
-inline bool ZAddress::is_good(uintptr_t value) {
-  return !is_bad(value) && !is_null(value);
+inline zoffset to_zoffset(uintptr_t value) {
+  assert(value < ZAddressOffsetMax, "must have no other bits");
+  return zoffset(value);
 }
 
-inline bool ZAddress::is_good_or_null(uintptr_t value) {
+inline zoffset to_zoffset(zoffset_end offset) {
+  const uintptr_t value = untype(offset);
+  return to_zoffset(value);
+}
+
+inline zoffset operator+(zoffset offset, size_t size) {
+  return to_zoffset(untype(offset) + size);
+}
+
+inline zoffset& operator+=(zoffset& offset, size_t size) {
+  offset = to_zoffset(untype(offset) + size);
+  return offset;
+}
+
+inline zoffset operator-(zoffset offset, size_t size) {
+  const uintptr_t value = untype(offset) - size;
+  return to_zoffset(value);
+}
+
+inline size_t operator-(zoffset left, zoffset right) {
+  const size_t diff = untype(left) - untype(right);
+  assert(diff < ZAddressOffsetMax, "Underflow");
+  return diff;
+}
+
+inline zoffset& operator-=(zoffset& offset, size_t size) {
+  offset = to_zoffset(untype(offset) - size);
+  return offset;
+}
+
+inline bool to_zoffset_end(zoffset_end* result, zoffset_end start, size_t size) {
+  const uintptr_t value = untype(start) + size;
+  if (value <= ZAddressOffsetMax) {
+    *result = zoffset_end(value);
+    return true;
+  }
+  return false;
+}
+
+inline zoffset_end to_zoffset_end(zoffset start, size_t size) {
+  const uintptr_t value = untype(start) + size;
+  assert(value <= ZAddressOffsetMax, "Overflow start: " PTR_FORMAT " size: " PTR_FORMAT " value: " PTR_FORMAT,
+                                     untype(start), size, value);
+  return zoffset_end(value);
+}
+
+inline zoffset_end to_zoffset_end(uintptr_t value) {
+  assert(value <= ZAddressOffsetMax, "Overflow");
+  return zoffset_end(value);
+}
+
+inline zoffset_end to_zoffset_end(zoffset offset) {
+  return zoffset_end(untype(offset));
+}
+
+inline bool operator!=(zoffset first, zoffset_end second) {
+  return untype(first) != untype(second);
+}
+
+inline bool operator!=(zoffset_end first, zoffset second) {
+  return untype(first) != untype(second);
+}
+
+inline bool operator==(zoffset first, zoffset_end second) {
+  return untype(first) == untype(second);
+}
+
+inline bool operator==(zoffset_end first, zoffset second) {
+  return untype(first) == untype(second);
+}
+
+inline bool operator<(zoffset_end first, zoffset second) {
+  return untype(first) < untype(second);
+}
+
+inline bool operator<(zoffset first, zoffset_end second) {
+  return untype(first) < untype(second);
+}
+
+inline bool operator>(zoffset first, zoffset_end second) {
+  return untype(first) > untype(second);
+}
+
+inline bool operator>=(zoffset first, zoffset_end second) {
+  return untype(first) >= untype(second);
+}
+
+inline size_t operator-(zoffset_end first, zoffset second) {
+  return untype(first) - untype(second);
+}
+
+inline zoffset_end operator-(zoffset_end first, size_t second) {
+  return to_zoffset_end(untype(first) - second);
+}
+
+inline size_t operator-(zoffset_end first, zoffset_end second) {
+  return untype(first) - untype(second);
+}
+
+inline zoffset_end& operator-=(zoffset_end& offset, size_t size) {
+  offset = to_zoffset_end(untype(offset) - size);
+  return offset;
+}
+
+inline zoffset_end& operator+=(zoffset_end& offset, size_t size) {
+  offset = to_zoffset_end(untype(offset) + size);
+  return offset;
+}
+
+// zpointer functions
+
+#define report_is_valid_failure(str) assert(!assert_on_failure, "%s: " PTR_FORMAT, str, value);
+
+inline bool is_valid(zpointer ptr, bool assert_on_failure = false) {
+  if (assert_on_failure && !ZVerifyOops) {
+    return true;
+  }
+
+  const uintptr_t value = static_cast<uintptr_t>(ptr);
+
+  if (value == 0) {
+    // Accept raw null
+    return false;
+  }
+
+  if ((value & ~ZPointerStoreMetadataMask) != 0) {
+#ifndef AARCH64
+    const int index = ZPointer::load_shift_lookup_index(value);
+    if (index != 0 && !is_power_of_2(index)) {
+      report_is_valid_failure("Invalid remap bits");
+      return false;
+    }
+#endif
+
+    const int shift = ZPointer::load_shift_lookup(value);
+    if (!is_power_of_2(value & (ZAddressHeapBase << shift))) {
+      report_is_valid_failure("Missing heap base");
+      return false;
+    }
+
+    if (((value >> shift) & 7) != 0) {
+      report_is_valid_failure("Alignment bits should not be set");
+      return false;
+    }
+  }
+
+  const uintptr_t load_metadata = ZPointer::remap_bits(value);
+  if (!is_power_of_2(load_metadata)) {
+    report_is_valid_failure("Must have exactly one load metadata bit");
+    return false;
+  }
+
+  const uintptr_t store_metadata = (value & (ZPointerStoreMetadataMask ^ ZPointerLoadMetadataMask));
+  const uintptr_t marked_young_metadata = store_metadata & (ZPointerMarkedYoung0 | ZPointerMarkedYoung1);
+  const uintptr_t marked_old_metadata = store_metadata & (ZPointerMarkedOld0 | ZPointerMarkedOld1 |
+                                                          ZPointerFinalizable0 | ZPointerFinalizable1);
+  const uintptr_t remembered_metadata = store_metadata & (ZPointerRemembered0 | ZPointerRemembered1);
+  if (!is_power_of_2(marked_young_metadata)) {
+    report_is_valid_failure("Must have exactly one marked young metadata bit");
+    return false;
+  }
+
+  if (!is_power_of_2(marked_old_metadata)) {
+    report_is_valid_failure("Must have exactly one marked old metadata bit");
+    return false;
+  }
+
+  if (remembered_metadata == 0) {
+    report_is_valid_failure("Must have at least one remembered metadata bit set");
+    return false;
+  }
+
+  if ((marked_young_metadata | marked_old_metadata | remembered_metadata) != store_metadata) {
+    report_is_valid_failure("Must have exactly three sets of store metadata bits");
+    return false;
+  }
+
+  if ((value & ZPointerReservedMask) != 0) {
+    report_is_valid_failure("Dirty reserved bits");
+    return false;
+  }
+
+  return true;
+}
+
+inline void assert_is_valid(zpointer ptr) {
+  DEBUG_ONLY(is_valid(ptr, true /* assert_on_failure */);)
+}
+
+inline uintptr_t untype(zpointer ptr) {
+  return static_cast<uintptr_t>(ptr);
+}
+
+inline zpointer to_zpointer(uintptr_t value) {
+  assert_is_valid(zpointer(value));
+  return zpointer(value);
+}
+
+inline zpointer to_zpointer(oopDesc* o) {
+  return ::to_zpointer(uintptr_t(o));
+}
+
+// Is it exactly null?
+inline bool is_null(zpointer ptr) {
+  return ptr == zpointer::null;
+}
+
+inline bool is_null_any(zpointer ptr) {
+  const uintptr_t raw_addr = untype(ptr);
+  return (raw_addr & ~ZPointerAllMetadataMask) == 0;
+}
+
+// Is it null - colored or not?
+inline bool is_null_assert_load_good(zpointer ptr) {
+  const bool result = is_null_any(ptr);
+  assert(!result || ZPointer::is_load_good(ptr), "Got bad colored null");
+  return result;
+}
+
+// zaddress functions
+
+inline bool is_null(zaddress addr) {
+  return addr == zaddress::null;
+}
+
+inline bool is_valid(zaddress addr, bool assert_on_failure = false) {
+  if (assert_on_failure && !ZVerifyOops) {
+    return true;
+  }
+
+  if (is_null(addr)) {
+    // Null is valid
+    return true;
+  }
+
+  const uintptr_t value = static_cast<uintptr_t>(addr);
+
+  if (value & 0x7) {
+    // No low order bits
+    report_is_valid_failure("Has low-order bits set");
+    return false;
+  }
+
+  if ((value & ZAddressHeapBase) == 0) {
+    // Must have a heap base bit
+    report_is_valid_failure("Missing heap base");
+    return false;
+  }
+
+  if (value >= (ZAddressHeapBase + ZAddressOffsetMax)) {
+    // Must not point outside of the heap's virtual address range
+    report_is_valid_failure("Address outside of the heap");
+    return false;
+  }
+
+  return true;
+}
+
+inline void assert_is_valid(zaddress addr) {
+  DEBUG_ONLY(is_valid(addr, true /* assert_on_failure */);)
+}
+
+inline uintptr_t untype(zaddress addr) {
+  return static_cast<uintptr_t>(addr);
+}
+
+#ifdef ASSERT
+inline void dereferenceable_test(zaddress addr) {
+  if (ZVerifyOops && !is_null(addr)) {
+    // Intentionally crash if the address is pointing into unmapped memory
+    (void)Atomic::load((int*)(uintptr_t)addr);
+  }
+}
+#endif
+
+inline zaddress to_zaddress(uintptr_t value) {
+  const zaddress addr = zaddress(value);
+  assert_is_valid(addr);
+  DEBUG_ONLY(dereferenceable_test(addr));
+  return addr;
+}
+
+inline zaddress to_zaddress(oopDesc* o) {
+  return to_zaddress(uintptr_t(o));
+}
+
+inline oop to_oop(zaddress addr) {
+  const oop obj = cast_to_oop(addr);
+  assert(!ZVerifyOops || oopDesc::is_oop_or_null(obj), "Broken oop: " PTR_FORMAT " [" PTR_FORMAT " " PTR_FORMAT " " PTR_FORMAT " " PTR_FORMAT "]",
+         p2i(obj),
+         *(uintptr_t*)(untype(addr) + 0x00),
+         *(uintptr_t*)(untype(addr) + 0x08),
+         *(uintptr_t*)(untype(addr) + 0x10),
+         *(uintptr_t*)(untype(addr) + 0x18));
+  return obj;
+}
+
+inline zaddress operator+(zaddress addr, size_t size) {
+  return to_zaddress(untype(addr) + size);
+}
+
+inline size_t operator-(zaddress left, zaddress right) {
+  assert(left >= right, "Unexpected order - left: " PTR_FORMAT " right: " PTR_FORMAT, untype(left), untype(right));
+  return untype(left) - untype(right);
+}
+
+// zaddress_unsafe functions
+
+inline bool is_null(zaddress_unsafe addr) {
+  return addr == zaddress_unsafe::null;
+}
+
+inline bool is_valid(zaddress_unsafe addr, bool assert_on_failure = false) {
+  return is_valid(zaddress(addr), assert_on_failure);
+}
+
+inline void assert_is_valid(zaddress_unsafe addr) {
+  DEBUG_ONLY(is_valid(addr, true /* assert_on_failure */);)
+}
+
+
+inline uintptr_t untype(zaddress_unsafe addr) {
+  return static_cast<uintptr_t>(addr);
+}
+
+// The zaddress_unsafe type denotes that this
+// memory isn't guaranteed to be dereferenceable.
+// The containing page could have been reclaimed
+// and/or uncommitted.
+//
+// The zaddress type denotes that this memory can
+// be dereferenced (runtime verified).
+//
+// This function can be used when the caller guarantees
+// that addr points to dereferenceable memory. Examples
+// of cases after which this function can be used:
+//
+// 1) A load good check on the colored pointer that addr was created from
+// 2) A load barrier has self-healed the pointer in addr
+// 3) A check that the addr doesn't belong to a relocation set. Since addr
+//    could denote two different objects in the two generations, a check
+//    against the colored pointer, that addr was created from, is needed to
+//    figure out what relocation set to look in.
+// 4) From the relocation code
+inline zaddress safe(zaddress_unsafe addr) {
+  return to_zaddress(untype(addr));
+}
+
+inline zaddress_unsafe to_zaddress_unsafe(uintptr_t value) {
+  const zaddress_unsafe addr = zaddress_unsafe(value);
+  assert_is_valid(addr);
+  return addr;
+}
+
+inline zaddress_unsafe unsafe(zaddress addr) {
+  return to_zaddress_unsafe(untype(addr));
+}
+
+inline zaddress_unsafe to_zaddress_unsafe(oop o) {
+  return to_zaddress_unsafe(cast_from_oop<uintptr_t>(o));
+}
+
+inline zaddress_unsafe operator+(zaddress_unsafe offset, size_t size) {
+  return to_zaddress_unsafe(untype(offset) + size);
+}
+
+inline size_t operator-(zaddress_unsafe left, zaddress_unsafe right) {
+  return untype(left) - untype(right);
+}
+
+// ZOffset functions
+
+inline zaddress ZOffset::address(zoffset offset) {
+  return to_zaddress(untype(offset) | ZAddressHeapBase);
+}
+
+inline zaddress_unsafe ZOffset::address_unsafe(zoffset offset) {
+  return to_zaddress_unsafe(untype(offset) | ZAddressHeapBase);
+}
+
+// ZPointer functions
+
+inline zaddress ZPointer::uncolor(zpointer ptr) {
+  assert(ZPointer::is_load_good(ptr) || is_null_any(ptr),
+      "Should be load good when handed out: " PTR_FORMAT, untype(ptr));
+  const uintptr_t raw_addr = untype(ptr);
+  return to_zaddress(raw_addr >> ZPointer::load_shift_lookup(raw_addr));
+}
+
+inline zaddress ZPointer::uncolor_store_good(zpointer ptr) {
+  assert(ZPointer::is_store_good(ptr), "Should be store good: " PTR_FORMAT, untype(ptr));
+  return uncolor(ptr);
+}
+
+inline zaddress_unsafe ZPointer::uncolor_unsafe(zpointer ptr) {
+  assert(ZPointer::is_store_bad(ptr), "Unexpected ptr");
+  const uintptr_t raw_addr = untype(ptr);
+  return to_zaddress_unsafe(raw_addr >> ZPointer::load_shift_lookup(raw_addr));
+}
+
+inline zpointer ZPointer::set_remset_bits(zpointer ptr) {
+  uintptr_t raw_addr = untype(ptr);
+  assert(raw_addr != 0, "raw nulls should have been purged in promotion to old gen");
+  raw_addr |= ZPointerRemembered0 | ZPointerRemembered1;
+  return to_zpointer(raw_addr);
+}
+
+inline bool ZPointer::is_load_bad(zpointer ptr) {
+  return untype(ptr) & ZPointerLoadBadMask;
+}
+
+inline bool ZPointer::is_load_good(zpointer ptr) {
+  return !is_load_bad(ptr) && !is_null(ptr);
+}
+
+inline bool ZPointer::is_load_good_or_null(zpointer ptr) {
   // Checking if an address is "not bad" is an optimized version of
   // checking if it's "good or null", which eliminates an explicit
   // null check. However, the implicit null check only checks that
@@ -51,87 +477,179 @@ inline bool ZAddress::is_good_or_null(uintptr_t value) {
   // This means that an address without mask bits would pass through
   // the barrier as if it was null. This should be harmless as such
   // addresses should ever be passed through the barrier.
-  const bool result = !is_bad(value);
-  assert((is_good(value) || is_null(value)) == result, "Bad address");
+  const bool result = !is_load_bad(ptr);
+  assert((is_load_good(ptr) || is_null(ptr)) == result, "Bad address");
   return result;
 }
 
-inline bool ZAddress::is_weak_bad(uintptr_t value) {
-  return value & ZAddressWeakBadMask;
+inline bool ZPointer::is_young_load_good(zpointer ptr) {
+  assert(!is_null(ptr), "not supported");
+  return (remap_bits(untype(ptr)) & ZPointerRemappedYoungMask) != 0;
 }
 
-inline bool ZAddress::is_weak_good(uintptr_t value) {
-  return !is_weak_bad(value) && !is_null(value);
+inline bool ZPointer::is_old_load_good(zpointer ptr) {
+  assert(!is_null(ptr), "not supported");
+  return (remap_bits(untype(ptr)) & ZPointerRemappedOldMask) != 0;
 }
 
-inline bool ZAddress::is_weak_good_or_null(uintptr_t value) {
-  return !is_weak_bad(value);
+inline bool ZPointer::is_mark_bad(zpointer ptr) {
+  return untype(ptr) & ZPointerMarkBadMask;
 }
 
-inline bool ZAddress::is_marked(uintptr_t value) {
-  return value & ZAddressMetadataMarked;
+inline bool ZPointer::is_mark_good(zpointer ptr) {
+  return !is_mark_bad(ptr) && !is_null(ptr);
 }
 
-inline bool ZAddress::is_marked_or_null(uintptr_t value) {
-  return is_marked(value) || is_null(value);
+inline bool ZPointer::is_mark_good_or_null(zpointer ptr) {
+  // Checking if an address is "not bad" is an optimized version of
+  // checking if it's "good or null", which eliminates an explicit
+  // null check. However, the implicit null check only checks that
+  // the mask bits are zero, not that the entire address is zero.
+  // This means that an address without mask bits would pass through
+  // the barrier as if it was null. This should be harmless as such
+  // addresses should ever be passed through the barrier.
+  const bool result = !is_mark_bad(ptr);
+  assert((is_mark_good(ptr) || is_null(ptr)) == result, "Bad address");
+  return result;
 }
 
-inline bool ZAddress::is_finalizable(uintptr_t value) {
-  return value & ZAddressMetadataFinalizable;
+inline bool ZPointer::is_store_bad(zpointer ptr) {
+  return untype(ptr) & ZPointerStoreBadMask;
 }
 
-inline bool ZAddress::is_finalizable_good(uintptr_t value) {
-  return is_finalizable(value) && is_good(value ^ ZAddressMetadataFinalizable);
+inline bool ZPointer::is_store_good(zpointer ptr) {
+  return !is_store_bad(ptr) && !is_null(ptr);
 }
 
-inline bool ZAddress::is_remapped(uintptr_t value) {
-  return value & ZAddressMetadataRemapped;
+inline bool ZPointer::is_store_good_or_null(zpointer ptr) {
+  // Checking if an address is "not bad" is an optimized version of
+  // checking if it's "good or null", which eliminates an explicit
+  // null check. However, the implicit null check only checks that
+  // the mask bits are zero, not that the entire address is zero.
+  // This means that an address without mask bits would pass through
+  // the barrier as if it was null. This should be harmless as such
+  // addresses should ever be passed through the barrier.
+  const bool result = !is_store_bad(ptr);
+  assert((is_store_good(ptr) || is_null(ptr)) == result, "Bad address");
+  return result;
 }
 
-inline bool ZAddress::is_in(uintptr_t value) {
-  // Check that exactly one non-offset bit is set
-  if (!is_power_of_2(value & ~ZAddressOffsetMask)) {
-    return false;
+inline bool ZPointer::is_marked_finalizable(zpointer ptr) {
+  assert(!is_null(ptr), "must not be null");
+  return untype(ptr) & ZPointerFinalizable;
+}
+
+inline bool ZPointer::is_marked_old(zpointer ptr) {
+  return untype(ptr) & (ZPointerMarkedOld);
+}
+
+inline bool ZPointer::is_marked_young(zpointer ptr) {
+  return untype(ptr) & (ZPointerMarkedYoung);
+}
+
+inline bool ZPointer::is_marked_any_old(zpointer ptr) {
+  return untype(ptr) & (ZPointerMarkedOld |
+                        ZPointerFinalizable);
+}
+
+inline bool ZPointer::is_remapped(zpointer ptr) {
+  assert(!is_null(ptr), "must not be null");
+  return remap_bits(untype(ptr)) & ZPointerRemapped;
+}
+
+inline bool ZPointer::is_remembered_exact(zpointer ptr) {
+  assert(!is_null(ptr), "must not be null");
+  return (untype(ptr) & ZPointerRemembered) == ZPointerRemembered;
+}
+
+inline constexpr int ZPointer::load_shift_lookup_index(uintptr_t value) {
+  return (value >> ZPointerRemappedShift) & ((1 << ZPointerRemappedBits) - 1);
+}
+
+// ZAddress functions
+
+inline zpointer ZAddress::color(zaddress addr, uintptr_t color) {
+  return to_zpointer((untype(addr) << ZPointer::load_shift_lookup(color)) | color);
+}
+
+inline zpointer ZAddress::color(zaddress_unsafe addr, uintptr_t color) {
+  return to_zpointer((untype(addr) << ZPointer::load_shift_lookup(color)) | color);
+}
+
+inline zoffset ZAddress::offset(zaddress addr) {
+  return to_zoffset(untype(addr) & ZAddressOffsetMask);
+}
+
+inline zoffset ZAddress::offset(zaddress_unsafe addr) {
+  return to_zoffset(untype(addr) & ZAddressOffsetMask);
+}
+
+inline zpointer color_null() {
+  return ZAddress::color(zaddress::null, ZPointerStoreGoodMask | ZPointerRememberedMask);
+}
+
+inline zpointer ZAddress::load_good(zaddress addr, zpointer prev) {
+  if (is_null_any(prev)) {
+    return color_null();
   }
 
-  // Check that one of the non-finalizable metadata is set
-  return value & (ZAddressMetadataMask & ~ZAddressMetadataFinalizable);
+  const uintptr_t non_load_bits_mask = ZPointerLoadMetadataMask ^ ZPointerAllMetadataMask;
+  const uintptr_t non_load_prev_bits = untype(prev) & non_load_bits_mask;
+  return color(addr, ZPointerLoadGoodMask | non_load_prev_bits | ZPointerRememberedMask);
 }
 
-inline uintptr_t ZAddress::offset(uintptr_t value) {
-  return value & ZAddressOffsetMask;
+inline zpointer ZAddress::finalizable_good(zaddress addr, zpointer prev) {
+  if (is_null_any(prev)) {
+    return color_null();
+  }
+
+  const uintptr_t non_mark_bits_mask = ZPointerMarkMetadataMask ^ ZPointerAllMetadataMask;
+  const uintptr_t non_mark_prev_bits = untype(prev) & non_mark_bits_mask;
+  return color(addr, ZPointerLoadGoodMask | ZPointerMarkedYoung | ZPointerFinalizable | non_mark_prev_bits | ZPointerRememberedMask);
 }
 
-inline uintptr_t ZAddress::good(uintptr_t value) {
-  return offset(value) | ZAddressGoodMask;
+inline zpointer ZAddress::mark_good(zaddress addr, zpointer prev) {
+  if (is_null_any(prev)) {
+    return color_null();
+  }
+
+  const uintptr_t non_mark_bits_mask = ZPointerMarkMetadataMask ^ ZPointerAllMetadataMask;
+  const uintptr_t non_mark_prev_bits = untype(prev) & non_mark_bits_mask;
+  return color(addr, ZPointerLoadGoodMask | ZPointerMarkedYoung | ZPointerMarkedOld | non_mark_prev_bits | ZPointerRememberedMask);
 }
 
-inline uintptr_t ZAddress::good_or_null(uintptr_t value) {
-  return is_null(value) ? 0 : good(value);
+inline zpointer ZAddress::mark_old_good(zaddress addr, zpointer prev) {
+  if (is_null_any(prev)) {
+    return color_null();
+  }
+
+  const uintptr_t prev_color = untype(prev);
+
+  const uintptr_t young_marked_mask = ZPointerMarkedYoung0 | ZPointerMarkedYoung1;
+  const uintptr_t young_marked = prev_color & young_marked_mask;
+
+  return color(addr, ZPointerLoadGoodMask | ZPointerMarkedOld | young_marked | ZPointerRememberedMask);
 }
 
-inline uintptr_t ZAddress::finalizable_good(uintptr_t value) {
-  return offset(value) | ZAddressMetadataFinalizable | ZAddressGoodMask;
+inline zpointer ZAddress::mark_young_good(zaddress addr, zpointer prev) {
+  if (is_null_any(prev)) {
+    return color_null();
+  }
+
+  const uintptr_t prev_color = untype(prev);
+
+  const uintptr_t old_marked_mask = ZPointerMarkedMask ^ (ZPointerMarkedYoung0 | ZPointerMarkedYoung1);
+  const uintptr_t old_marked = prev_color & old_marked_mask;
+
+  return color(addr, ZPointerLoadGoodMask | ZPointerMarkedYoung | old_marked | ZPointerRememberedMask);
 }
 
-inline uintptr_t ZAddress::marked(uintptr_t value) {
-  return offset(value) | ZAddressMetadataMarked;
+inline zpointer ZAddress::store_good(zaddress addr) {
+  return color(addr, ZPointerStoreGoodMask);
 }
 
-inline uintptr_t ZAddress::marked0(uintptr_t value) {
-  return offset(value) | ZAddressMetadataMarked0;
-}
-
-inline uintptr_t ZAddress::marked1(uintptr_t value) {
-  return offset(value) | ZAddressMetadataMarked1;
-}
-
-inline uintptr_t ZAddress::remapped(uintptr_t value) {
-  return offset(value) | ZAddressMetadataRemapped;
-}
-
-inline uintptr_t ZAddress::remapped_or_null(uintptr_t value) {
-  return is_null(value) ? 0 : remapped(value);
+inline zpointer ZAddress::store_good_or_null(zaddress addr) {
+  return is_null(addr) ? zpointer::null : store_good(addr);
 }
 
 #endif // SHARE_GC_Z_ZADDRESS_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zAddressSpaceLimit.cpp b/src/hotspot/share/gc/z/zAddressSpaceLimit.cpp
index 518fa07e0d4..41ae19566b9 100644
--- a/src/hotspot/share/gc/z/zAddressSpaceLimit.cpp
+++ b/src/hotspot/share/gc/z/zAddressSpaceLimit.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -46,8 +46,8 @@ size_t ZAddressSpaceLimit::mark_stack() {
   return align_up(limit, ZMarkStackSpaceExpandSize);
 }
 
-size_t ZAddressSpaceLimit::heap_view() {
-  // Allow all heap views to occupy 50% of the address space
-  const size_t limit = address_space_limit() / MaxVirtMemFraction / ZHeapViews;
+size_t ZAddressSpaceLimit::heap() {
+  // Allow the heap to occupy 50% of the address space
+  const size_t limit = address_space_limit() / MaxVirtMemFraction;
   return align_up(limit, ZGranuleSize);
 }
diff --git a/src/hotspot/share/gc/z/zAddressSpaceLimit.hpp b/src/hotspot/share/gc/z/zAddressSpaceLimit.hpp
index ec0faf2c087..d8e7e7cfd36 100644
--- a/src/hotspot/share/gc/z/zAddressSpaceLimit.hpp
+++ b/src/hotspot/share/gc/z/zAddressSpaceLimit.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,7 +30,7 @@
 class ZAddressSpaceLimit : public AllStatic {
 public:
   static size_t mark_stack();
-  static size_t heap_view();
+  static size_t heap();
 };
 
 #endif // SHARE_GC_Z_ZADDRESSSPACELIMIT_HPP
diff --git a/src/hotspot/share/gc/z/zAllocationFlags.hpp b/src/hotspot/share/gc/z/zAllocationFlags.hpp
index 86b195efafd..44d6ea740ea 100644
--- a/src/hotspot/share/gc/z/zAllocationFlags.hpp
+++ b/src/hotspot/share/gc/z/zAllocationFlags.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -38,7 +38,7 @@
 //  |     | | |
 //  |     | | * 0-0 Non-Blocking Flag (1-bit)
 //  |     | |
-//  |     | * 1-1 Worker Relocation Flag (1-bit)
+//  |     | * 1-1 GC Relocation Flag (1-bit)
 //  |     |
 //  |     * 2-2 Low Address Flag (1-bit)
 //  |
@@ -48,7 +48,7 @@
 class ZAllocationFlags {
 private:
   typedef ZBitField<uint8_t, bool, 0, 1> field_non_blocking;
-  typedef ZBitField<uint8_t, bool, 1, 1> field_worker_relocation;
+  typedef ZBitField<uint8_t, bool, 1, 1> field_gc_relocation;
   typedef ZBitField<uint8_t, bool, 2, 1> field_low_address;
 
   uint8_t _flags;
@@ -61,8 +61,8 @@ public:
     _flags |= field_non_blocking::encode(true);
   }
 
-  void set_worker_relocation() {
-    _flags |= field_worker_relocation::encode(true);
+  void set_gc_relocation() {
+    _flags |= field_gc_relocation::encode(true);
   }
 
   void set_low_address() {
@@ -73,8 +73,8 @@ public:
     return field_non_blocking::decode(_flags);
   }
 
-  bool worker_relocation() const {
-    return field_worker_relocation::decode(_flags);
+  bool gc_relocation() const {
+    return field_gc_relocation::decode(_flags);
   }
 
   bool low_address() const {
diff --git a/src/hotspot/share/gc/z/zAllocator.cpp b/src/hotspot/share/gc/z/zAllocator.cpp
new file mode 100644
index 00000000000..9fdb811a89a
--- /dev/null
+++ b/src/hotspot/share/gc/z/zAllocator.cpp
@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zAllocator.hpp"
+#include "gc/z/zObjectAllocator.hpp"
+
+ZAllocatorEden*          ZAllocator::_eden;
+ZAllocatorForRelocation* ZAllocator::_relocation[ZAllocator::_relocation_allocators];
+
+ZAllocator::ZAllocator(ZPageAge age) :
+    _object_allocator(age) {}
+
+void ZAllocator::retire_pages() {
+  _object_allocator.retire_pages();
+}
+
+ZAllocatorEden::ZAllocatorEden() :
+    ZAllocator(ZPageAge::eden) {
+  ZAllocator::_eden = this;
+}
+
+size_t ZAllocatorEden::tlab_used() const {
+  return _object_allocator.used();
+}
+
+size_t ZAllocatorEden::remaining() const {
+  return _object_allocator.remaining();
+}
+
+ZPageAge ZAllocatorForRelocation::install() {
+  for (uint i = 0; i < ZAllocator::_relocation_allocators; ++i) {
+    if (_relocation[i] == nullptr) {
+      _relocation[i] = this;
+      return static_cast<ZPageAge>(i + 1);
+    }
+  }
+
+  ShouldNotReachHere();
+  return ZPageAge::eden;
+}
+
+ZAllocatorForRelocation::ZAllocatorForRelocation() :
+    ZAllocator(install()) {
+}
+
+zaddress ZAllocatorForRelocation::alloc_object(size_t size) {
+  return _object_allocator.alloc_object_for_relocation(size);
+}
+
+void ZAllocatorForRelocation::undo_alloc_object(zaddress addr, size_t size) {
+  _object_allocator.undo_alloc_object_for_relocation(addr, size);
+}
+
+ZPage* ZAllocatorForRelocation::alloc_page_for_relocation(ZPageType type, size_t size, ZAllocationFlags flags) {
+  return _object_allocator.alloc_page_for_relocation(type, size, flags);
+}
diff --git a/src/hotspot/share/gc/z/zAllocator.hpp b/src/hotspot/share/gc/z/zAllocator.hpp
new file mode 100644
index 00000000000..3457f2b3c18
--- /dev/null
+++ b/src/hotspot/share/gc/z/zAllocator.hpp
@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZALLOCATOR_HPP
+#define SHARE_GC_Z_ZALLOCATOR_HPP
+
+#include "gc/z/zAllocationFlags.hpp"
+#include "gc/z/zObjectAllocator.hpp"
+#include "gc/z/zPageAge.hpp"
+#include "gc/z/zPageType.hpp"
+
+class ZAllocatorEden;
+class ZAllocatorForRelocation;
+class ZPage;
+
+class ZAllocator {
+  friend class ZAllocatorEden;
+  friend class ZAllocatorSurvivor;
+  friend class ZAllocatorOld;
+
+public:
+  static constexpr uint _relocation_allocators = static_cast<uint>(ZPageAge::old);
+
+protected:
+  ZObjectAllocator _object_allocator;
+
+  static ZAllocatorEden*          _eden;
+  static ZAllocatorForRelocation* _relocation[ZAllocator::_relocation_allocators];
+
+public:
+  static ZAllocatorEden* eden();
+  static ZAllocatorForRelocation* relocation(ZPageAge page_age);
+  static ZAllocatorForRelocation* old();
+
+  ZAllocator(ZPageAge age);
+
+  void retire_pages();
+};
+
+class ZAllocatorEden : public ZAllocator {
+public:
+  ZAllocatorEden();
+
+  // Mutator allocation
+  zaddress alloc_tlab(size_t size);
+  zaddress alloc_object(size_t size);
+
+  // Statistics
+  size_t tlab_used() const;
+  size_t remaining() const;
+};
+
+class ZAllocatorForRelocation : public ZAllocator {
+private:
+  ZPageAge install();
+
+public:
+  ZAllocatorForRelocation();
+
+  // Relocation
+  zaddress alloc_object(size_t size);
+  void undo_alloc_object(zaddress addr, size_t size);
+
+  ZPage* alloc_page_for_relocation(ZPageType type, size_t size, ZAllocationFlags flags);
+};
+
+#endif // SHARE_GC_Z_ZALLOCATOR_HPP
diff --git a/src/hotspot/share/gc/z/zAllocator.inline.hpp b/src/hotspot/share/gc/z/zAllocator.inline.hpp
new file mode 100644
index 00000000000..ba558a1b0f3
--- /dev/null
+++ b/src/hotspot/share/gc/z/zAllocator.inline.hpp
@@ -0,0 +1,59 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZALLOCATOR_INLINE_HPP
+#define SHARE_GC_Z_ZALLOCATOR_INLINE_HPP
+
+#include "gc/z/zAllocator.hpp"
+
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zHeap.hpp"
+
+inline ZAllocatorEden* ZAllocator::eden() {
+  return _eden;
+}
+
+inline ZAllocatorForRelocation* ZAllocator::relocation(ZPageAge page_age) {
+  return _relocation[static_cast<uint>(page_age) - 1];
+}
+
+inline ZAllocatorForRelocation* ZAllocator::old() {
+  return relocation(ZPageAge::old);
+}
+
+inline zaddress ZAllocatorEden::alloc_tlab(size_t size) {
+  guarantee(size <= ZHeap::heap()->max_tlab_size(), "TLAB too large");
+  return _object_allocator.alloc_object(size);
+}
+
+inline zaddress ZAllocatorEden::alloc_object(size_t size) {
+  const zaddress addr = _object_allocator.alloc_object(size);
+
+  if (is_null(addr)) {
+    ZHeap::heap()->out_of_memory();
+  }
+
+  return addr;
+}
+
+#endif // SHARE_GC_Z_ZALLOCATOR_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zArguments.cpp b/src/hotspot/share/gc/z/zArguments.cpp
index ad59d2fbcec..01ecf8f3fc4 100644
--- a/src/hotspot/share/gc/z/zArguments.cpp
+++ b/src/hotspot/share/gc/z/zArguments.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -37,9 +37,74 @@ void ZArguments::initialize_alignments() {
   HeapAlignment = SpaceAlignment;
 }
 
-void ZArguments::initialize() {
-  GCArguments::initialize();
+void ZArguments::select_max_gc_threads() {
+  // Select number of parallel threads
+  if (FLAG_IS_DEFAULT(ParallelGCThreads)) {
+    FLAG_SET_DEFAULT(ParallelGCThreads, ZHeuristics::nparallel_workers());
+  }
 
+  if (ParallelGCThreads == 0) {
+    vm_exit_during_initialization("The flag -XX:+UseZGC can not be combined with -XX:ParallelGCThreads=0");
+  }
+
+  // The max number of concurrent threads we heuristically want for a generation
+  uint max_nworkers_generation;
+
+  if (FLAG_IS_DEFAULT(ConcGCThreads)) {
+    max_nworkers_generation = ZHeuristics::nconcurrent_workers();
+
+    // Computed max number of GC threads at a time in the machine
+    uint max_nworkers = max_nworkers_generation;
+
+    if (!FLAG_IS_DEFAULT(ZYoungGCThreads)) {
+      max_nworkers = MAX2(max_nworkers, ZYoungGCThreads);
+    }
+
+    if (!FLAG_IS_DEFAULT(ZOldGCThreads)) {
+      max_nworkers = MAX2(max_nworkers, ZOldGCThreads);
+    }
+
+    FLAG_SET_DEFAULT(ConcGCThreads, max_nworkers);
+  } else {
+    max_nworkers_generation = ConcGCThreads;
+  }
+
+  if (FLAG_IS_DEFAULT(ZYoungGCThreads)) {
+    if (UseDynamicNumberOfGCThreads) {
+      FLAG_SET_ERGO(ZYoungGCThreads, max_nworkers_generation);
+    } else {
+      const uint static_young_threads = MAX2(uint(max_nworkers_generation * 0.9), 1u);
+      FLAG_SET_ERGO(ZYoungGCThreads, static_young_threads);
+    }
+  }
+
+  if (FLAG_IS_DEFAULT(ZOldGCThreads)) {
+    if (UseDynamicNumberOfGCThreads) {
+      FLAG_SET_ERGO(ZOldGCThreads, max_nworkers_generation);
+    } else {
+      const uint static_old_threads = MAX2(ConcGCThreads - ZYoungGCThreads, 1u);
+      FLAG_SET_ERGO(ZOldGCThreads, static_old_threads);
+    }
+  }
+
+  if (ConcGCThreads == 0) {
+    vm_exit_during_initialization("The flag -XX:+UseZGC can not be combined with -XX:ConcGCThreads=0");
+  }
+
+  if (ZYoungGCThreads > ConcGCThreads) {
+    vm_exit_during_initialization("The flag -XX:ZYoungGCThreads can't be higher than -XX:ConcGCThreads");
+  } else if (ZYoungGCThreads == 0) {
+    vm_exit_during_initialization("The flag -XX:ZYoungGCThreads can't be lower than 1");
+  }
+
+  if (ZOldGCThreads > ConcGCThreads) {
+    vm_exit_during_initialization("The flag -XX:ZOldGCThreads can't be higher than -XX:ConcGCThreads");
+  } else if (ZOldGCThreads == 0) {
+    vm_exit_during_initialization("The flag -XX:ZOldGCThreads can't be lower than 1");
+  }
+}
+
+void ZArguments::initialize() {
   // Check mark stack size
   const size_t mark_stack_space_limit = ZAddressSpaceLimit::mark_stack();
   if (ZMarkStackSpaceLimit > mark_stack_space_limit) {
@@ -54,22 +119,53 @@ void ZArguments::initialize() {
     FLAG_SET_DEFAULT(UseNUMA, true);
   }
 
-  // Select number of parallel threads
-  if (FLAG_IS_DEFAULT(ParallelGCThreads)) {
-    FLAG_SET_DEFAULT(ParallelGCThreads, ZHeuristics::nparallel_workers());
+  select_max_gc_threads();
+
+  // Backwards compatible alias for ZCollectionIntervalMajor
+  if (!FLAG_IS_DEFAULT(ZCollectionInterval)) {
+    FLAG_SET_ERGO_IF_DEFAULT(ZCollectionIntervalMajor, ZCollectionInterval);
   }
 
-  if (ParallelGCThreads == 0) {
-    vm_exit_during_initialization("The flag -XX:+UseZGC can not be combined with -XX:ParallelGCThreads=0");
+  if (!FLAG_IS_CMDLINE(MaxHeapSize) &&
+      !FLAG_IS_CMDLINE(MaxRAMFraction) &&
+      !FLAG_IS_CMDLINE(MaxRAMPercentage)) {
+    // We are really just guessing how much memory the program needs.
+    // When that is the case, we don't want the soft and hard limits to be the same
+    // as it can cause flakyness in the number of GC threads used, in order to keep
+    // to a random number we just pulled out of thin air.
+    FLAG_SET_ERGO_IF_DEFAULT(SoftMaxHeapSize, MaxHeapSize * 90 / 100);
   }
 
-  // Select number of concurrent threads
-  if (FLAG_IS_DEFAULT(ConcGCThreads)) {
-    FLAG_SET_DEFAULT(ConcGCThreads, ZHeuristics::nconcurrent_workers());
+  if (FLAG_IS_DEFAULT(ZFragmentationLimit)) {
+    FLAG_SET_DEFAULT(ZFragmentationLimit, 5.0);
   }
 
-  if (ConcGCThreads == 0) {
-    vm_exit_during_initialization("The flag -XX:+UseZGC can not be combined with -XX:ConcGCThreads=0");
+  if (!FLAG_IS_DEFAULT(ZTenuringThreshold) && ZTenuringThreshold != -1) {
+    FLAG_SET_ERGO_IF_DEFAULT(MaxTenuringThreshold, ZTenuringThreshold);
+    if (MaxTenuringThreshold == 0) {
+      FLAG_SET_ERGO_IF_DEFAULT(AlwaysTenure, true);
+    }
+  }
+
+  if (FLAG_IS_DEFAULT(MaxTenuringThreshold)) {
+    uint tenuring_threshold;
+    for (tenuring_threshold = 0; tenuring_threshold < MaxTenuringThreshold; ++tenuring_threshold) {
+      // Reduce the number of object ages, if the resulting garbage is too high
+      const size_t medium_page_overhead = ZPageSizeMedium * tenuring_threshold;
+      const size_t small_page_overhead = ZPageSizeSmall * ConcGCThreads * tenuring_threshold;
+      if (small_page_overhead + medium_page_overhead >= ZHeuristics::significant_young_overhead()) {
+        break;
+      }
+    }
+    FLAG_SET_DEFAULT(MaxTenuringThreshold, tenuring_threshold);
+    if (tenuring_threshold == 0 && FLAG_IS_DEFAULT(AlwaysTenure)) {
+      // Some flag constraint function says AlwaysTenure must be true iff MaxTenuringThreshold == 0
+      FLAG_SET_DEFAULT(AlwaysTenure, true);
+    }
+  }
+
+  if (!FLAG_IS_DEFAULT(ZTenuringThreshold) && NeverTenure) {
+    vm_exit_during_initialization(err_msg("ZTenuringThreshold and NeverTenure are incompatible"));
   }
 
   // Large page size must match granule size
@@ -79,10 +175,9 @@ void ZArguments::initialize() {
                                           ZGranuleSize / M));
   }
 
-  // The heuristics used when UseDynamicNumberOfGCThreads is
-  // enabled defaults to using a ZAllocationSpikeTolerance of 1.
-  if (UseDynamicNumberOfGCThreads && FLAG_IS_DEFAULT(ZAllocationSpikeTolerance)) {
-    FLAG_SET_DEFAULT(ZAllocationSpikeTolerance, 1);
+  if (!FLAG_IS_DEFAULT(ZTenuringThreshold) && ZTenuringThreshold > static_cast<int>(MaxTenuringThreshold)) {
+    vm_exit_during_initialization(err_msg("ZTenuringThreshold must be be within bounds of "
+                                          "MaxTenuringThreshold"));
   }
 
 #ifdef COMPILER2
@@ -98,6 +193,11 @@ void ZArguments::initialize() {
   // CompressedOops not supported
   FLAG_SET_DEFAULT(UseCompressedOops, false);
 
+  // More events
+  if (FLAG_IS_DEFAULT(LogEventsBufferEntries)) {
+    FLAG_SET_DEFAULT(LogEventsBufferEntries, 250);
+  }
+
   // Verification before startup and after exit not (yet) supported
   FLAG_SET_DEFAULT(VerifyDuringStartup, false);
   FLAG_SET_DEFAULT(VerifyBeforeExit, false);
@@ -106,20 +206,23 @@ void ZArguments::initialize() {
     FLAG_SET_DEFAULT(ZVerifyRoots, true);
     FLAG_SET_DEFAULT(ZVerifyObjects, true);
   }
+
+#ifdef ASSERT
+  // This check slows down testing too much. Turn it off for now.
+  if (FLAG_IS_DEFAULT(VerifyDependencies)) {
+    FLAG_SET_DEFAULT(VerifyDependencies, false);
+  }
+#endif
 }
 
 size_t ZArguments::heap_virtual_to_physical_ratio() {
-  return ZHeapViews * ZVirtualToPhysicalRatio;
-}
-
-size_t ZArguments::conservative_max_heap_alignment() {
-  return 0;
+  return ZVirtualToPhysicalRatio;
 }
 
 CollectedHeap* ZArguments::create_heap() {
   return new ZCollectedHeap();
 }
 
-bool ZArguments::is_supported() const {
+bool ZArguments::is_supported() {
   return is_os_supported();
 }
diff --git a/src/hotspot/share/gc/z/zArguments.hpp b/src/hotspot/share/gc/z/zArguments.hpp
index cc177e25d90..ac1e613d4cc 100644
--- a/src/hotspot/share/gc/z/zArguments.hpp
+++ b/src/hotspot/share/gc/z/zArguments.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,18 +28,19 @@
 
 class CollectedHeap;
 
-class ZArguments : public GCArguments {
+class ZArguments : AllStatic {
 private:
-  virtual void initialize_alignments();
+  static void select_max_gc_threads();
 
-  virtual void initialize();
-  virtual size_t conservative_max_heap_alignment();
-  virtual size_t heap_virtual_to_physical_ratio();
-  virtual CollectedHeap* create_heap();
+public:
+  static void initialize_alignments();
+  static void initialize();
+  static size_t heap_virtual_to_physical_ratio();
+  static CollectedHeap* create_heap();
 
-  virtual bool is_supported() const;
+  static bool is_supported();
 
-  bool is_os_supported() const;
+  static bool is_os_supported();
 };
 
 #endif // SHARE_GC_Z_ZARGUMENTS_HPP
diff --git a/src/hotspot/share/gc/z/zArray.hpp b/src/hotspot/share/gc/z/zArray.hpp
index f7bd3967d79..7bcd4f59eeb 100644
--- a/src/hotspot/share/gc/z/zArray.hpp
+++ b/src/hotspot/share/gc/z/zArray.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,27 +25,59 @@
 #define SHARE_GC_Z_ZARRAY_HPP
 
 #include "memory/allocation.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/os.hpp"
+#include "runtime/thread.hpp"
 #include "utilities/growableArray.hpp"
 
+#include <type_traits>
+
+class ZLock;
+
 template <typename T> using ZArray = GrowableArrayCHeap<T, mtGC>;
 
 template <typename T, bool Parallel>
 class ZArrayIteratorImpl : public StackObj {
 private:
-  const T*       _next;
-  const T* const _end;
+  size_t         _next;
+  const size_t   _end;
+  const T* const _array;
 
-  bool next_serial(T* elem);
-  bool next_parallel(T* elem);
+  bool next_serial(size_t* index);
+  bool next_parallel(size_t* index);
 
 public:
   ZArrayIteratorImpl(const T* array, size_t length);
   ZArrayIteratorImpl(const ZArray<T>* array);
 
   bool next(T* elem);
+  bool next_index(size_t* index);
+
+  T index_to_elem(size_t index);
 };
 
 template <typename T> using ZArrayIterator = ZArrayIteratorImpl<T, false /* Parallel */>;
 template <typename T> using ZArrayParallelIterator = ZArrayIteratorImpl<T, true /* Parallel */>;
 
+template <typename T>
+class ZActivatedArray {
+private:
+  typedef typename std::remove_extent<T>::type ItemT;
+
+  ZLock*         _lock;
+  uint64_t       _count;
+  ZArray<ItemT*> _array;
+
+public:
+  explicit ZActivatedArray(bool locked = true);
+  ~ZActivatedArray();
+
+  void activate();
+  template <typename Function>
+  void deactivate_and_apply(Function function);
+
+  bool is_activated() const;
+  bool add_if_activated(ItemT* item);
+};
+
 #endif // SHARE_GC_Z_ZARRAY_HPP
diff --git a/src/hotspot/share/gc/z/zArray.inline.hpp b/src/hotspot/share/gc/z/zArray.inline.hpp
index b7e5bba9adb..d38945750d2 100644
--- a/src/hotspot/share/gc/z/zArray.inline.hpp
+++ b/src/hotspot/share/gc/z/zArray.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,55 +26,121 @@
 
 #include "gc/z/zArray.hpp"
 
+#include "gc/z/zLock.inline.hpp"
 #include "runtime/atomic.hpp"
 
 template <typename T, bool Parallel>
-inline bool ZArrayIteratorImpl<T, Parallel>::next_serial(T* elem) {
+inline bool ZArrayIteratorImpl<T, Parallel>::next_serial(size_t* index) {
   if (_next == _end) {
     return false;
   }
 
-  *elem = *_next;
+  *index = _next;
   _next++;
 
   return true;
 }
 
 template <typename T, bool Parallel>
-inline bool ZArrayIteratorImpl<T, Parallel>::next_parallel(T* elem) {
-  const T* old_next = Atomic::load(&_next);
+inline bool ZArrayIteratorImpl<T, Parallel>::next_parallel(size_t* index) {
+  const size_t claimed_index = Atomic::fetch_and_add(&_next, 1u, memory_order_relaxed);
 
-  for (;;) {
-    if (old_next == _end) {
-      return false;
-    }
-
-    const T* const new_next = old_next + 1;
-    const T* const prev_next = Atomic::cmpxchg(&_next, old_next, new_next);
-    if (prev_next == old_next) {
-      *elem = *old_next;
-      return true;
-    }
-
-    old_next = prev_next;
+  if (claimed_index < _end) {
+    *index = claimed_index;
+    return true;
   }
+
+  return false;
 }
 
 template <typename T, bool Parallel>
 inline ZArrayIteratorImpl<T, Parallel>::ZArrayIteratorImpl(const T* array, size_t length) :
-    _next(array),
-    _end(array + length) {}
+    _next(0),
+    _end(length),
+    _array(array) {}
 
 template <typename T, bool Parallel>
 inline ZArrayIteratorImpl<T, Parallel>::ZArrayIteratorImpl(const ZArray<T>* array) :
-    ZArrayIteratorImpl<T, Parallel>(array->is_empty() ? NULL : array->adr_at(0), array->length()) {}
+    ZArrayIteratorImpl<T, Parallel>(array->is_empty() ? nullptr : array->adr_at(0), array->length()) {}
 
 template <typename T, bool Parallel>
 inline bool ZArrayIteratorImpl<T, Parallel>::next(T* elem) {
+  size_t index;
+  if (next_index(&index)) {
+    *elem = index_to_elem(index);
+    return true;
+  }
+
+  return false;
+}
+
+template <typename T, bool Parallel>
+inline bool ZArrayIteratorImpl<T, Parallel>::next_index(size_t* index) {
   if (Parallel) {
-    return next_parallel(elem);
+    return next_parallel(index);
   } else {
-    return next_serial(elem);
+    return next_serial(index);
+  }
+}
+
+template <typename T, bool Parallel>
+inline T ZArrayIteratorImpl<T, Parallel>::index_to_elem(size_t index) {
+  assert(index < _end, "Out of bounds");
+  return _array[index];
+}
+
+template <typename T>
+ZActivatedArray<T>::ZActivatedArray(bool locked) :
+    _lock(locked ? new ZLock() : nullptr),
+    _count(0),
+    _array() {}
+
+template <typename T>
+ZActivatedArray<T>::~ZActivatedArray<T>() {
+  FreeHeap(_lock);
+}
+
+template <typename T>
+bool ZActivatedArray<T>::is_activated() const {
+  ZLocker<ZLock> locker(_lock);
+  return _count > 0;
+}
+
+template <typename T>
+bool ZActivatedArray<T>::add_if_activated(ItemT* item) {
+  ZLocker<ZLock> locker(_lock);
+  if (_count > 0) {
+    _array.append(item);
+    return true;
+  }
+
+  return false;
+}
+
+template <typename T>
+void ZActivatedArray<T>::activate() {
+  ZLocker<ZLock> locker(_lock);
+  _count++;
+}
+
+template <typename T>
+template <typename Function>
+void ZActivatedArray<T>::deactivate_and_apply(Function function) {
+  ZArray<ItemT*> array;
+
+  {
+    ZLocker<ZLock> locker(_lock);
+    assert(_count > 0, "Invalid state");
+    if (--_count == 0u) {
+      // Fully deactivated - remove all elements
+      array.swap(&_array);
+    }
+  }
+
+  // Apply function to all elements - if fully deactivated
+  ZArrayIterator<ItemT*> iter(&array);
+  for (ItemT* item; iter.next(&item);) {
+    function(item);
   }
 }
 
diff --git a/src/hotspot/share/gc/z/zBarrier.cpp b/src/hotspot/share/gc/z/zBarrier.cpp
index 63730c58b19..6e1d3ee0093 100644
--- a/src/hotspot/share/gc/z/zBarrier.cpp
+++ b/src/hotspot/share/gc/z/zBarrier.cpp
@@ -23,253 +23,291 @@
 
 #include "precompiled.hpp"
 #include "classfile/javaClasses.hpp"
+#include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zBarrier.inline.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zHeap.inline.hpp"
-#include "gc/z/zOop.inline.hpp"
-#include "gc/z/zThread.inline.hpp"
+#include "gc/z/zStoreBarrierBuffer.inline.hpp"
 #include "memory/iterator.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "runtime/safepoint.hpp"
 #include "utilities/debug.hpp"
 
-template <bool finalizable>
-bool ZBarrier::should_mark_through(uintptr_t addr) {
-  // Finalizable marked oops can still exists on the heap after marking
-  // has completed, in which case we just want to convert this into a
-  // good oop and not push it on the mark stack.
-  if (!during_mark()) {
-    assert(ZAddress::is_marked(addr), "Should be marked");
-    assert(ZAddress::is_finalizable(addr), "Should be finalizable");
-    return false;
-  }
-
-  // During marking, we mark through already marked oops to avoid having
-  // some large part of the object graph hidden behind a pushed, but not
-  // yet flushed, entry on a mutator mark stack. Always marking through
-  // allows the GC workers to proceed through the object graph even if a
-  // mutator touched an oop first, which in turn will reduce the risk of
-  // having to flush mark stacks multiple times to terminate marking.
-  //
-  // However, when doing finalizable marking we don't always want to mark
-  // through. First, marking through an already strongly marked oop would
-  // be wasteful, since we will then proceed to do finalizable marking on
-  // an object which is, or will be, marked strongly. Second, marking
-  // through an already finalizable marked oop would also be wasteful,
-  // since such oops can never end up on a mutator mark stack and can
-  // therefore not hide some part of the object graph from GC workers.
-  if (finalizable) {
-    return !ZAddress::is_marked(addr);
-  }
-
-  // Mark through
-  return true;
+#ifdef ASSERT
+static bool during_young_mark() {
+  return ZGeneration::young()->is_phase_mark();
 }
 
-template <bool gc_thread, bool follow, bool finalizable, bool publish>
-uintptr_t ZBarrier::mark(uintptr_t addr) {
-  uintptr_t good_addr;
-
-  if (ZAddress::is_marked(addr)) {
-    // Already marked, but try to mark though anyway
-    good_addr = ZAddress::good(addr);
-  } else if (ZAddress::is_remapped(addr)) {
-    // Already remapped, but also needs to be marked
-    good_addr = ZAddress::good(addr);
-  } else {
-    // Needs to be both remapped and marked
-    good_addr = remap(addr);
-  }
-
-  // Mark
-  if (should_mark_through<finalizable>(addr)) {
-    ZHeap::heap()->mark_object<gc_thread, follow, finalizable, publish>(good_addr);
-  }
-
-  if (finalizable) {
-    // Make the oop finalizable marked/good, instead of normal marked/good.
-    // This is needed because an object might first becomes finalizable
-    // marked by the GC, and then loaded by a mutator thread. In this case,
-    // the mutator thread must be able to tell that the object needs to be
-    // strongly marked. The finalizable bit in the oop exists to make sure
-    // that a load of a finalizable marked oop will fall into the barrier
-    // slow path so that we can mark the object as strongly reachable.
-    return ZAddress::finalizable_good(good_addr);
-  }
-
-  return good_addr;
+static bool during_old_mark() {
+  return ZGeneration::old()->is_phase_mark();
 }
 
-uintptr_t ZBarrier::remap(uintptr_t addr) {
-  assert(!ZAddress::is_good(addr), "Should not be good");
-  assert(!ZAddress::is_weak_good(addr), "Should not be weak good");
-  return ZHeap::heap()->remap_object(addr);
+static bool during_any_mark() {
+  return during_young_mark() || during_old_mark();
+}
+#endif
+
+zaddress ZBarrier::relocate_or_remap(zaddress_unsafe addr, ZGeneration* generation) {
+  return generation->relocate_or_remap_object(addr);
 }
 
-uintptr_t ZBarrier::relocate(uintptr_t addr) {
-  assert(!ZAddress::is_good(addr), "Should not be good");
-  assert(!ZAddress::is_weak_good(addr), "Should not be weak good");
-  return ZHeap::heap()->relocate_object(addr);
-}
-
-uintptr_t ZBarrier::relocate_or_mark(uintptr_t addr) {
-  return during_relocate() ? relocate(addr) : mark<AnyThread, Follow, Strong, Publish>(addr);
-}
-
-uintptr_t ZBarrier::relocate_or_mark_no_follow(uintptr_t addr) {
-  return during_relocate() ? relocate(addr) : mark<AnyThread, DontFollow, Strong, Publish>(addr);
-}
-
-uintptr_t ZBarrier::relocate_or_remap(uintptr_t addr) {
-  return during_relocate() ? relocate(addr) : remap(addr);
-}
-
-//
-// Load barrier
-//
-uintptr_t ZBarrier::load_barrier_on_oop_slow_path(uintptr_t addr) {
-  return relocate_or_mark(addr);
-}
-
-uintptr_t ZBarrier::load_barrier_on_invisible_root_oop_slow_path(uintptr_t addr) {
-  return relocate_or_mark_no_follow(addr);
-}
-
-void ZBarrier::load_barrier_on_oop_fields(oop o) {
-  assert(ZAddress::is_good(ZOop::to_address(o)), "Should be good");
-  ZLoadBarrierOopClosure cl;
-  o->oop_iterate(&cl);
+zaddress ZBarrier::remap(zaddress_unsafe addr, ZGeneration* generation) {
+  return generation->remap_object(addr);
 }
 
 //
 // Weak load barrier
 //
-uintptr_t ZBarrier::weak_load_barrier_on_oop_slow_path(uintptr_t addr) {
-  return ZAddress::is_weak_good(addr) ? ZAddress::good(addr) : relocate_or_remap(addr);
+
+static void keep_alive_young(zaddress addr) {
+  if (ZGeneration::young()->is_phase_mark()) {
+    ZBarrier::mark_young<ZMark::Resurrect, ZMark::AnyThread, ZMark::Follow>(addr);
+  }
 }
 
-uintptr_t ZBarrier::weak_load_barrier_on_weak_oop_slow_path(uintptr_t addr) {
-  const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr);
-  if (ZHeap::heap()->is_object_strongly_live(good_addr)) {
-    return good_addr;
+zaddress ZBarrier::blocking_keep_alive_on_weak_slow_path(volatile zpointer* p, zaddress addr) {
+  if (is_null(addr)) {
+    return zaddress::null;
   }
 
-  // Not strongly live
-  return 0;
+  if (ZHeap::heap()->is_old(addr)) {
+    if (!ZHeap::heap()->is_object_strongly_live(addr)) {
+      return zaddress::null;
+    }
+  } else {
+    // Young gen objects are never blocked, need to keep alive
+    keep_alive_young(addr);
+  }
+
+  // Strongly live
+  return addr;
 }
 
-uintptr_t ZBarrier::weak_load_barrier_on_phantom_oop_slow_path(uintptr_t addr) {
-  const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr);
-  if (ZHeap::heap()->is_object_live(good_addr)) {
-    return good_addr;
+zaddress ZBarrier::blocking_keep_alive_on_phantom_slow_path(volatile zpointer* p, zaddress addr) {
+  if (is_null(addr)) {
+    return zaddress::null;
   }
 
-  // Not live
-  return 0;
+  if (ZHeap::heap()->is_old(addr)) {
+    if (!ZHeap::heap()->is_object_live(addr)) {
+      return zaddress::null;
+    }
+  } else {
+    // Young gen objects are never blocked, need to keep alive
+    keep_alive_young(addr);
+  }
+
+  // Strongly live
+  return addr;
+}
+
+zaddress ZBarrier::blocking_load_barrier_on_weak_slow_path(volatile zpointer* p, zaddress addr) {
+  if (is_null(addr)) {
+    return zaddress::null;
+  }
+
+  if (ZHeap::heap()->is_old(addr)) {
+    if (!ZHeap::heap()->is_object_strongly_live(addr)) {
+      return zaddress::null;
+    }
+  } else {
+    // Young objects are never considered non-strong
+    // Note: Should not need to keep object alive in this operation,
+    //       but the barrier colors the pointer mark good, so we need
+    //       to mark the object accordingly.
+    keep_alive_young(addr);
+  }
+
+  return addr;
+}
+
+zaddress ZBarrier::blocking_load_barrier_on_phantom_slow_path(volatile zpointer* p, zaddress addr) {
+  if (is_null(addr)) {
+    return zaddress::null;
+  }
+
+  if (ZHeap::heap()->is_old(addr)) {
+    if (!ZHeap::heap()->is_object_live(addr)) {
+      return zaddress::null;
+    }
+  } else {
+    // Young objects are never considered non-strong
+    // Note: Should not need to keep object alive in this operation,
+    //       but the barrier colors the pointer mark good, so we need
+    //       to mark the object accordingly.
+    keep_alive_young(addr);
+  }
+
+  return addr;
 }
 
 //
-// Keep alive barrier
+// Clean barrier
 //
-uintptr_t ZBarrier::keep_alive_barrier_on_oop_slow_path(uintptr_t addr) {
-  assert(during_mark(), "Invalid phase");
 
-  // Mark
-  return mark<AnyThread, Follow, Strong, Overflow>(addr);
-}
+zaddress ZBarrier::verify_old_object_live_slow_path(zaddress addr) {
+  // Verify that the object was indeed alive
+  assert(ZHeap::heap()->is_young(addr) || ZHeap::heap()->is_object_live(addr), "Should be live");
 
-uintptr_t ZBarrier::keep_alive_barrier_on_weak_oop_slow_path(uintptr_t addr) {
-  assert(ZResurrection::is_blocked(), "This operation is only valid when resurrection is blocked");
-  const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr);
-  assert(ZHeap::heap()->is_object_strongly_live(good_addr), "Should be live");
-  return good_addr;
-}
-
-uintptr_t ZBarrier::keep_alive_barrier_on_phantom_oop_slow_path(uintptr_t addr) {
-  assert(ZResurrection::is_blocked(), "This operation is only valid when resurrection is blocked");
-  const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr);
-  assert(ZHeap::heap()->is_object_live(good_addr), "Should be live");
-  return good_addr;
+  return addr;
 }
 
 //
 // Mark barrier
 //
-uintptr_t ZBarrier::mark_barrier_on_oop_slow_path(uintptr_t addr) {
-  assert(during_mark(), "Invalid phase");
-  assert(ZThread::is_worker(), "Invalid thread");
 
-  // Mark
-  return mark<GCThread, Follow, Strong, Overflow>(addr);
+zaddress ZBarrier::mark_slow_path(zaddress addr) {
+  assert(during_any_mark(), "Invalid phase");
+
+  if (is_null(addr)) {
+    return addr;
+  }
+
+  mark<ZMark::DontResurrect, ZMark::AnyThread, ZMark::Follow, ZMark::Strong>(addr);
+
+  return addr;
 }
 
-uintptr_t ZBarrier::mark_barrier_on_finalizable_oop_slow_path(uintptr_t addr) {
-  assert(during_mark(), "Invalid phase");
-  assert(ZThread::is_worker(), "Invalid thread");
+zaddress ZBarrier::mark_from_young_slow_path(zaddress addr) {
+  assert(during_young_mark(), "Invalid phase");
 
-  // Mark
-  return mark<GCThread, Follow, Finalizable, Overflow>(addr);
+  if (is_null(addr)) {
+    return addr;
+  }
+
+  if (ZHeap::heap()->is_young(addr)) {
+    ZGeneration::young()->mark_object<ZMark::DontResurrect, ZMark::GCThread, ZMark::Follow, ZMark::Strong>(addr);
+    return addr;
+  }
+
+  if (ZGeneration::young()->type() == ZYoungType::major_full_roots ||
+      ZGeneration::young()->type() == ZYoungType::major_partial_roots) {
+    // The initial major young collection is responsible for finding roots
+    // from the young generation to the old generation.
+    ZGeneration::old()->mark_object<ZMark::DontResurrect, ZMark::GCThread, ZMark::Follow, ZMark::Strong>(addr);
+    return addr;
+  }
+
+  // Don't mark pointers to the old generation for minor during major;
+  // the initial young collection pushed the young-to-old pointers that
+  // were part of the SATB. All other young-to-old pointers are irrelevant.
+
+  // We still want to heal the pointers so they become store_good, so that
+  // after a young collection, all young pointers are store good.
+  return addr;
 }
 
-//
-// Narrow oop variants, never used.
-//
-oop ZBarrier::load_barrier_on_oop_field(volatile narrowOop* p) {
-  ShouldNotReachHere();
-  return NULL;
+zaddress ZBarrier::mark_from_old_slow_path(zaddress addr) {
+  assert(during_old_mark(), "Invalid phase");
+
+  if (is_null(addr)) {
+    return addr;
+  }
+
+  if (ZHeap::heap()->is_old(addr)) {
+    ZGeneration::old()->mark_object<ZMark::DontResurrect, ZMark::GCThread, ZMark::Follow, ZMark::Strong>(addr);
+    return addr;
+  }
+
+  // Don't mark pointers to the young generation; they will be
+  // processed by the remembered set scanning.
+
+  // Returning null means this location is not self healed by the caller.
+  return zaddress::null;
 }
 
-oop ZBarrier::load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o) {
-  ShouldNotReachHere();
-  return NULL;
+zaddress ZBarrier::mark_young_slow_path(zaddress addr) {
+  assert(during_young_mark(), "Invalid phase");
+
+  if (is_null(addr)) {
+    return addr;
+  }
+
+  mark_if_young<ZMark::DontResurrect, ZMark::GCThread, ZMark::Follow>(addr);
+
+  return addr;
 }
 
-void ZBarrier::load_barrier_on_oop_array(volatile narrowOop* p, size_t length) {
-  ShouldNotReachHere();
+zaddress ZBarrier::mark_finalizable_slow_path(zaddress addr) {
+  assert(during_any_mark(), "Invalid phase");
+
+  if (is_null(addr)) {
+    return addr;
+  }
+
+  if (ZHeap::heap()->is_old(addr)) {
+    ZGeneration::old()->mark_object<ZMark::DontResurrect, ZMark::GCThread, ZMark::Follow, ZMark::Finalizable>(addr);
+    return addr;
+  }
+
+  ZGeneration::young()->mark_object_if_active<ZMark::DontResurrect, ZMark::GCThread, ZMark::Follow, ZMark::Strong>(addr);
+  return addr;
 }
 
-oop ZBarrier::load_barrier_on_weak_oop_field_preloaded(volatile narrowOop* p, oop o) {
-  ShouldNotReachHere();
-  return NULL;
+zaddress ZBarrier::mark_finalizable_from_old_slow_path(zaddress addr) {
+  assert(during_any_mark(), "Invalid phase");
+
+  if (is_null(addr)) {
+    return addr;
+  }
+
+  if (ZHeap::heap()->is_old(addr)) {
+    ZGeneration::old()->mark_object<ZMark::DontResurrect, ZMark::GCThread, ZMark::Follow, ZMark::Finalizable>(addr);
+    return addr;
+  }
+
+  // Don't mark pointers to the young generation; they will be
+  // processed by the remembered set scanning.
+
+  // Returning null means this location is not self healed by the caller.
+  return zaddress::null;
 }
 
-oop ZBarrier::load_barrier_on_phantom_oop_field_preloaded(volatile narrowOop* p, oop o) {
-  ShouldNotReachHere();
-  return NULL;
+zaddress ZBarrier::heap_store_slow_path(volatile zpointer* p, zaddress addr, zpointer prev, bool heal) {
+  ZStoreBarrierBuffer* buffer = ZStoreBarrierBuffer::buffer_for_store(heal);
+
+  if (buffer != nullptr) {
+    // Buffer store barriers whenever possible
+    buffer->add(p, prev);
+  } else {
+    mark_and_remember(p, addr);
+  }
+
+  return addr;
 }
 
-oop ZBarrier::weak_load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o) {
-  ShouldNotReachHere();
-  return NULL;
+zaddress ZBarrier::no_keep_alive_heap_store_slow_path(volatile zpointer* p, zaddress addr) {
+  remember(p);
+
+  return addr;
 }
 
-oop ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(volatile narrowOop* p, oop o) {
-  ShouldNotReachHere();
-  return NULL;
+zaddress ZBarrier::native_store_slow_path(zaddress addr) {
+  if (!is_null(addr)) {
+    mark<ZMark::DontResurrect, ZMark::AnyThread, ZMark::Follow, ZMark::Strong>(addr);
+  }
+
+  return addr;
 }
 
-oop ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(volatile narrowOop* p, oop o) {
-  ShouldNotReachHere();
-  return NULL;
+zaddress ZBarrier::keep_alive_slow_path(zaddress addr) {
+  if (!is_null(addr)) {
+    mark<ZMark::Resurrect, ZMark::AnyThread, ZMark::Follow, ZMark::Strong>(addr);
+  }
+
+  return addr;
 }
 
 #ifdef ASSERT
 
 // ON_WEAK barriers should only ever be applied to j.l.r.Reference.referents.
-void ZBarrier::verify_on_weak(volatile oop* referent_addr) {
-  if (referent_addr != NULL) {
-    uintptr_t base = (uintptr_t)referent_addr - java_lang_ref_Reference::referent_offset();
-    oop obj = cast_to_oop(base);
+void ZBarrier::verify_on_weak(volatile zpointer* referent_addr) {
+  if (referent_addr != nullptr) {
+    const uintptr_t base = (uintptr_t)referent_addr - java_lang_ref_Reference::referent_offset();
+    const oop obj = cast_to_oop(base);
     assert(oopDesc::is_oop(obj), "Verification failed for: ref " PTR_FORMAT " obj: " PTR_FORMAT, (uintptr_t)referent_addr, base);
     assert(java_lang_ref_Reference::is_referent_field(obj, java_lang_ref_Reference::referent_offset()), "Sanity");
   }
 }
 
 #endif
-
-void ZLoadBarrierOopClosure::do_oop(oop* p) {
-  ZBarrier::load_barrier_on_oop_field(p);
-}
-
-void ZLoadBarrierOopClosure::do_oop(narrowOop* p) {
-  ShouldNotReachHere();
-}
diff --git a/src/hotspot/share/gc/z/zBarrier.hpp b/src/hotspot/share/gc/z/zBarrier.hpp
index 2dfc1591888..14ad0a655f7 100644
--- a/src/hotspot/share/gc/z/zBarrier.hpp
+++ b/src/hotspot/share/gc/z/zBarrier.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,112 +24,162 @@
 #ifndef SHARE_GC_Z_ZBARRIER_HPP
 #define SHARE_GC_Z_ZBARRIER_HPP
 
+#include "gc/z/zAddress.hpp"
 #include "memory/allStatic.hpp"
 #include "memory/iterator.hpp"
-#include "oops/oop.hpp"
 
-typedef bool (*ZBarrierFastPath)(uintptr_t);
-typedef uintptr_t (*ZBarrierSlowPath)(uintptr_t);
+// == Shift based load barrier ==
+//
+// The load barriers of ZGC check if a loaded value is safe to expose or not, and
+// then shifts the pointer to remove metadata bits, such that it points to mapped
+// memory.
+//
+// A pointer is safe to expose if it does not have any load-bad bits set in its
+// metadata bits. In the C++ code and non-nmethod generated code, that is checked
+// by testing the pointer value against a load-bad mask, checking that no bad bit
+// is set, followed by a shift, removing the metadata bits if they were good.
+// However, for nmethod code, the test + shift sequence is optimized in such
+// a way that the shift both tests if the pointer is exposable or not, and removes
+// the metadata bits, with the same instruction. This is a speculative optimization
+// that assumes that the loaded pointer is frequently going to be load-good or null
+// when checked. Therefore, the nmethod load barriers just apply the shift with the
+// current "good" shift (which is patched with nmethod entry barriers for each GC
+// phase). If the result of that shift was a raw null value, then the ZF flag is set.
+// If the result is a good pointer, then the very last bit that was removed by the
+// shift, must have been a 1, which would have set the CF flag. Therefore, the "above"
+// branch condition code is used to take a slowpath only iff CF == 0 and ZF == 0.
+// CF == 0 implies it was not a good pointer, and ZF == 0 implies the resulting address
+// was not a null value. Then we decide that the pointer is bad. This optimization
+// is necessary to get satisfactory performance, but does come with a few constraints:
+//
+// 1) The load barrier can only recognize 4 different good patterns across all GC phases.
+//    The reason is that when a load barrier applies the currently good shift, then
+//    the value of said shift may differ only by 3, until we risk shifting away more
+//    than the low order three zeroes of an address, given a bad pointer, which would
+//    yield spurious false positives.
+//
+// 2) Those bit patterns must have only a single bit set. We achieve that by moving
+//    non-relocation work to store barriers.
+//
+// Another consequence of this speculative optimization, is that when the compiled code
+// takes a slow path, it needs to reload the oop, because the shifted oop is now
+// broken after being shifted with a different shift to what was used when the oop
+// was stored.
+
+typedef bool (*ZBarrierFastPath)(zpointer);
+typedef zpointer (*ZBarrierColor)(zaddress, zpointer);
+
+class ZGeneration;
+
+void z_assert_is_barrier_safe();
 
 class ZBarrier : public AllStatic {
+  friend class ZContinuation;
+  friend class ZStoreBarrierBuffer;
+  friend class ZUncoloredRoot;
+
 private:
-  static const bool GCThread    = true;
-  static const bool AnyThread   = false;
+  static void assert_transition_monotonicity(zpointer ptr, zpointer heal_ptr);
+  static void self_heal(ZBarrierFastPath fast_path, volatile zpointer* p, zpointer ptr, zpointer heal_ptr, bool allow_null);
 
-  static const bool Follow      = true;
-  static const bool DontFollow  = false;
+  template <typename ZBarrierSlowPath>
+  static zaddress barrier(ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path, ZBarrierColor color, volatile zpointer* p, zpointer o, bool allow_null = false);
 
-  static const bool Strong      = false;
-  static const bool Finalizable = true;
+  static zaddress make_load_good(zpointer ptr);
+  static zaddress make_load_good_no_relocate(zpointer ptr);
+  static zaddress relocate_or_remap(zaddress_unsafe addr, ZGeneration* generation);
+  static zaddress remap(zaddress_unsafe addr, ZGeneration* generation);
+  static void remember(volatile zpointer* p);
+  static void mark_and_remember(volatile zpointer* p, zaddress addr);
 
-  static const bool Publish     = true;
-  static const bool Overflow    = false;
+  // Fast paths in increasing strength level
+  static bool is_load_good_or_null_fast_path(zpointer ptr);
+  static bool is_mark_good_fast_path(zpointer ptr);
+  static bool is_store_good_fast_path(zpointer ptr);
+  static bool is_store_good_or_null_fast_path(zpointer ptr);
+  static bool is_store_good_or_null_any_fast_path(zpointer ptr);
 
-  template <ZBarrierFastPath fast_path> static void self_heal(volatile oop* p, uintptr_t addr, uintptr_t heal_addr);
+  static bool is_mark_young_good_fast_path(zpointer ptr);
+  static bool is_finalizable_good_fast_path(zpointer ptr);
 
-  template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path> static oop barrier(volatile oop* p, oop o);
-  template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path> static oop weak_barrier(volatile oop* p, oop o);
-  template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path> static void root_barrier(oop* p, oop o);
+  // Slow paths
+  static zaddress blocking_keep_alive_on_weak_slow_path(volatile zpointer* p, zaddress addr);
+  static zaddress blocking_keep_alive_on_phantom_slow_path(volatile zpointer* p, zaddress addr);
+  static zaddress blocking_load_barrier_on_weak_slow_path(volatile zpointer* p, zaddress addr);
+  static zaddress blocking_load_barrier_on_phantom_slow_path(volatile zpointer* p, zaddress addr);
 
-  static bool is_good_or_null_fast_path(uintptr_t addr);
-  static bool is_weak_good_or_null_fast_path(uintptr_t addr);
-  static bool is_marked_or_null_fast_path(uintptr_t addr);
+  static zaddress verify_old_object_live_slow_path(zaddress addr);
 
-  static bool during_mark();
-  static bool during_relocate();
-  template <bool finalizable> static bool should_mark_through(uintptr_t addr);
-  template <bool gc_thread, bool follow, bool finalizable, bool publish> static uintptr_t mark(uintptr_t addr);
-  static uintptr_t remap(uintptr_t addr);
-  static uintptr_t relocate(uintptr_t addr);
-  static uintptr_t relocate_or_mark(uintptr_t addr);
-  static uintptr_t relocate_or_mark_no_follow(uintptr_t addr);
-  static uintptr_t relocate_or_remap(uintptr_t addr);
+  static zaddress mark_slow_path(zaddress addr);
+  static zaddress mark_young_slow_path(zaddress addr);
+  static zaddress mark_from_young_slow_path(zaddress addr);
+  static zaddress mark_from_old_slow_path(zaddress addr);
+  static zaddress mark_finalizable_slow_path(zaddress addr);
+  static zaddress mark_finalizable_from_old_slow_path(zaddress addr);
 
-  static uintptr_t load_barrier_on_oop_slow_path(uintptr_t addr);
-  static uintptr_t load_barrier_on_invisible_root_oop_slow_path(uintptr_t addr);
+  static zaddress keep_alive_slow_path(zaddress addr);
+  static zaddress heap_store_slow_path(volatile zpointer* p, zaddress addr, zpointer prev, bool heal);
+  static zaddress native_store_slow_path(zaddress addr);
+  static zaddress no_keep_alive_heap_store_slow_path(volatile zpointer* p, zaddress addr);
 
-  static uintptr_t weak_load_barrier_on_oop_slow_path(uintptr_t addr);
-  static uintptr_t weak_load_barrier_on_weak_oop_slow_path(uintptr_t addr);
-  static uintptr_t weak_load_barrier_on_phantom_oop_slow_path(uintptr_t addr);
+  static zaddress promote_slow_path(zaddress addr);
 
-  static uintptr_t keep_alive_barrier_on_oop_slow_path(uintptr_t addr);
-  static uintptr_t keep_alive_barrier_on_weak_oop_slow_path(uintptr_t addr);
-  static uintptr_t keep_alive_barrier_on_phantom_oop_slow_path(uintptr_t addr);
+  // Helpers for non-strong oop refs barriers
+  static zaddress blocking_keep_alive_load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o);
+  static zaddress blocking_keep_alive_load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o);
+  static zaddress blocking_load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o);
+  static zaddress blocking_load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o);
 
-  static uintptr_t mark_barrier_on_oop_slow_path(uintptr_t addr);
-  static uintptr_t mark_barrier_on_finalizable_oop_slow_path(uintptr_t addr);
-
-  static void verify_on_weak(volatile oop* referent_addr) NOT_DEBUG_RETURN;
+  // Verification
+  static void verify_on_weak(volatile zpointer* referent_addr) NOT_DEBUG_RETURN;
 
 public:
+
+  static zpointer load_atomic(volatile zpointer* p);
+
+  // Helpers for relocation
+  static ZGeneration* remap_generation(zpointer ptr);
+  static void remap_young_relocated(volatile zpointer* p, zpointer o);
+
+  // Helpers for marking
+  template <bool resurrect, bool gc_thread, bool follow, bool finalizable>
+  static void mark(zaddress addr);
+  template <bool resurrect, bool gc_thread, bool follow>
+  static void mark_young(zaddress addr);
+  template <bool resurrect, bool gc_thread, bool follow>
+  static void mark_if_young(zaddress addr);
+
   // Load barrier
-  static  oop load_barrier_on_oop(oop o);
-  static  oop load_barrier_on_oop_field(volatile oop* p);
-  static  oop load_barrier_on_oop_field_preloaded(volatile oop* p, oop o);
-  static void load_barrier_on_oop_array(volatile oop* p, size_t length);
-  static void load_barrier_on_oop_fields(oop o);
-  static  oop load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o);
-  static  oop load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o);
-  static void load_barrier_on_root_oop_field(oop* p);
-  static void load_barrier_on_invisible_root_oop_field(oop* p);
+  static zaddress load_barrier_on_oop_field(volatile zpointer* p);
+  static zaddress load_barrier_on_oop_field_preloaded(volatile zpointer* p, zpointer o);
 
-  // Weak load barrier
-  static oop weak_load_barrier_on_oop_field(volatile oop* p);
-  static oop weak_load_barrier_on_oop_field_preloaded(volatile oop* p, oop o);
-  static oop weak_load_barrier_on_weak_oop(oop o);
-  static oop weak_load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o);
-  static oop weak_load_barrier_on_phantom_oop(oop o);
-  static oop weak_load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o);
+  static zaddress keep_alive_load_barrier_on_oop_field_preloaded(volatile zpointer* p, zpointer o);
 
-  // Is alive barrier
-  static bool is_alive_barrier_on_weak_oop(oop o);
-  static bool is_alive_barrier_on_phantom_oop(oop o);
+  // Load barriers on non-strong oop refs
+  static zaddress load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o);
+  static zaddress load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o);
 
-  // Keep alive barrier
-  static void keep_alive_barrier_on_oop(oop o);
-  static void keep_alive_barrier_on_weak_oop_field(volatile oop* p);
-  static void keep_alive_barrier_on_phantom_oop_field(volatile oop* p);
-  static void keep_alive_barrier_on_phantom_root_oop_field(oop* p);
+  static zaddress no_keep_alive_load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o);
+  static zaddress no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o);
+
+  // Reference processor / weak cleaning barriers
+  static bool clean_barrier_on_weak_oop_field(volatile zpointer* p);
+  static bool clean_barrier_on_phantom_oop_field(volatile zpointer* p);
+  static bool clean_barrier_on_final_oop_field(volatile zpointer* p);
 
   // Mark barrier
-  static void mark_barrier_on_oop_field(volatile oop* p, bool finalizable);
-  static void mark_barrier_on_oop_array(volatile oop* p, size_t length, bool finalizable);
+  static void mark_barrier_on_young_oop_field(volatile zpointer* p);
+  static void mark_barrier_on_old_oop_field(volatile zpointer* p, bool finalizable);
+  static void mark_barrier_on_oop_field(volatile zpointer* p, bool finalizable);
+  static void mark_young_good_barrier_on_oop_field(volatile zpointer* p);
+  static zaddress remset_barrier_on_oop_field(volatile zpointer* p);
+  static void promote_barrier_on_young_oop_field(volatile zpointer* p);
 
-  // Narrow oop variants, never used.
-  static oop  load_barrier_on_oop_field(volatile narrowOop* p);
-  static oop  load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o);
-  static void load_barrier_on_oop_array(volatile narrowOop* p, size_t length);
-  static oop  load_barrier_on_weak_oop_field_preloaded(volatile narrowOop* p, oop o);
-  static oop  load_barrier_on_phantom_oop_field_preloaded(volatile narrowOop* p, oop o);
-  static oop  weak_load_barrier_on_oop_field_preloaded(volatile narrowOop* p, oop o);
-  static oop  weak_load_barrier_on_weak_oop_field_preloaded(volatile narrowOop* p, oop o);
-  static oop  weak_load_barrier_on_phantom_oop_field_preloaded(volatile narrowOop* p, oop o);
-};
+  // Store barrier
+  static void store_barrier_on_heap_oop_field(volatile zpointer* p, bool heal);
+  static void store_barrier_on_native_oop_field(volatile zpointer* p, bool heal);
 
-class ZLoadBarrierOopClosure : public BasicOopIterateClosure {
-public:
-  virtual void do_oop(oop* p);
-  virtual void do_oop(narrowOop* p);
+  static void no_keep_alive_store_barrier_on_heap_oop_field(volatile zpointer* p);
 };
 
 #endif // SHARE_GC_Z_ZBARRIER_HPP
diff --git a/src/hotspot/share/gc/z/zBarrier.inline.hpp b/src/hotspot/share/gc/z/zBarrier.inline.hpp
index 391dd09a8c9..e0d83619934 100644
--- a/src/hotspot/share/gc/z/zBarrier.inline.hpp
+++ b/src/hotspot/share/gc/z/zBarrier.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,85 +28,50 @@
 
 #include "code/codeCache.hpp"
 #include "gc/z/zAddress.inline.hpp"
-#include "gc/z/zOop.inline.hpp"
+#include "gc/z/zGeneration.inline.hpp"
+#include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zResurrection.inline.hpp"
 #include "oops/oop.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/continuation.hpp"
 
 // A self heal must always "upgrade" the address metadata bits in
-// accordance with the metadata bits state machine, which has the
-// valid state transitions as described below (where N is the GC
-// cycle).
-//
-// Note the subtleness of overlapping GC cycles. Specifically that
-// oops are colored Remapped(N) starting at relocation N and ending
-// at marking N + 1.
-//
-//              +--- Mark Start
-//              | +--- Mark End
-//              | | +--- Relocate Start
-//              | | | +--- Relocate End
-//              | | | |
-// Marked       |---N---|--N+1--|--N+2--|----
-// Finalizable  |---N---|--N+1--|--N+2--|----
-// Remapped     ----|---N---|--N+1--|--N+2--|
-//
-// VALID STATE TRANSITIONS
-//
-//   Marked(N)           -> Remapped(N)
-//                       -> Marked(N + 1)
-//                       -> Finalizable(N + 1)
-//
-//   Finalizable(N)      -> Marked(N)
-//                       -> Remapped(N)
-//                       -> Marked(N + 1)
-//                       -> Finalizable(N + 1)
-//
-//   Remapped(N)         -> Marked(N + 1)
-//                       -> Finalizable(N + 1)
-//
-// PHASE VIEW
-//
-// ZPhaseMark
-//   Load & Mark
-//     Marked(N)         <- Marked(N - 1)
-//                       <- Finalizable(N - 1)
-//                       <- Remapped(N - 1)
-//                       <- Finalizable(N)
-//
-//   Mark(Finalizable)
-//     Finalizable(N)    <- Marked(N - 1)
-//                       <- Finalizable(N - 1)
-//                       <- Remapped(N - 1)
-//
-//   Load(AS_NO_KEEPALIVE)
-//     Remapped(N - 1)   <- Marked(N - 1)
-//                       <- Finalizable(N - 1)
-//
-// ZPhaseMarkCompleted (Resurrection blocked)
-//   Load & Load(ON_WEAK/PHANTOM_OOP_REF | AS_NO_KEEPALIVE) & KeepAlive
-//     Marked(N)         <- Marked(N - 1)
-//                       <- Finalizable(N - 1)
-//                       <- Remapped(N - 1)
-//                       <- Finalizable(N)
-//
-//   Load(ON_STRONG_OOP_REF | AS_NO_KEEPALIVE)
-//     Remapped(N - 1)   <- Marked(N - 1)
-//                       <- Finalizable(N - 1)
-//
-// ZPhaseMarkCompleted (Resurrection unblocked)
-//   Load
-//     Marked(N)         <- Finalizable(N)
-//
-// ZPhaseRelocate
-//   Load & Load(AS_NO_KEEPALIVE)
-//     Remapped(N)       <- Marked(N)
-//                       <- Finalizable(N)
+// accordance with the metadata bits state machine. The following
+// assert verifies the monotonicity of the transitions.
 
-template <ZBarrierFastPath fast_path>
-inline void ZBarrier::self_heal(volatile oop* p, uintptr_t addr, uintptr_t heal_addr) {
-  if (heal_addr == 0) {
+inline void ZBarrier::assert_transition_monotonicity(zpointer old_ptr, zpointer new_ptr) {
+  const bool old_is_load_good = ZPointer::is_load_good(old_ptr);
+  const bool old_is_mark_good = ZPointer::is_mark_good(old_ptr);
+  const bool old_is_store_good = ZPointer::is_store_good(old_ptr);
+
+  const bool new_is_load_good = ZPointer::is_load_good(new_ptr);
+  const bool new_is_mark_good = ZPointer::is_mark_good(new_ptr);
+  const bool new_is_store_good = ZPointer::is_store_good(new_ptr);
+
+  assert(!old_is_load_good || new_is_load_good, "non-monotonic load good transition");
+  assert(!old_is_mark_good || new_is_mark_good, "non-monotonic mark good transition");
+  assert(!old_is_store_good || new_is_store_good, "non-monotonic store good transition");
+
+  if (is_null_any(new_ptr)) {
+    // Null is good enough at this point
+    return;
+  }
+
+  const bool old_is_marked_young = ZPointer::is_marked_young(old_ptr);
+  const bool old_is_marked_old = ZPointer::is_marked_old(old_ptr);
+  const bool old_is_marked_finalizable = ZPointer::is_marked_finalizable(old_ptr);
+
+  const bool new_is_marked_young = ZPointer::is_marked_young(new_ptr);
+  const bool new_is_marked_old = ZPointer::is_marked_old(new_ptr);
+  const bool new_is_marked_finalizable = ZPointer::is_marked_finalizable(new_ptr);
+
+  assert(!old_is_marked_young || new_is_marked_young, "non-monotonic marked young transition");
+  assert(!old_is_marked_old || new_is_marked_old, "non-monotonic marked old transition");
+  assert(!old_is_marked_finalizable || new_is_marked_finalizable || new_is_marked_old, "non-monotonic marked final transition");
+}
+
+inline void ZBarrier::self_heal(ZBarrierFastPath fast_path, volatile zpointer* p, zpointer ptr, zpointer heal_ptr, bool allow_null) {
+  if (!allow_null && is_null_assert_load_good(heal_ptr) && !is_null_any(ptr)) {
     // Never heal with null since it interacts badly with reference processing.
     // A mutator clearing an oop would be similar to calling Reference.clear(),
     // which would make the reference non-discoverable or silently dropped
@@ -114,18 +79,28 @@ inline void ZBarrier::self_heal(volatile oop* p, uintptr_t addr, uintptr_t heal_
     return;
   }
 
-  assert(!fast_path(addr), "Invalid self heal");
-  assert(fast_path(heal_addr), "Invalid self heal");
+  assert_is_valid(ptr);
+  assert_is_valid(heal_ptr);
+  assert(!fast_path(ptr), "Invalid self heal");
+  assert(fast_path(heal_ptr), "Invalid self heal");
+
+  assert(ZPointer::is_remapped(heal_ptr), "invariant");
 
   for (;;) {
+    if (ptr == zpointer::null) {
+      assert(!ZVerifyOops || !ZHeap::heap()->is_in(uintptr_t(p)) || !ZHeap::heap()->is_old(p), "No raw null in old");
+    }
+
+    assert_transition_monotonicity(ptr, heal_ptr);
+
     // Heal
-    const uintptr_t prev_addr = Atomic::cmpxchg((volatile uintptr_t*)p, addr, heal_addr, memory_order_relaxed);
-    if (prev_addr == addr) {
+    const zpointer prev_ptr = Atomic::cmpxchg(p, ptr, heal_ptr, memory_order_relaxed);
+    if (prev_ptr == ptr) {
       // Success
       return;
     }
 
-    if (fast_path(prev_addr)) {
+    if (fast_path(prev_ptr)) {
       // Must not self heal
       return;
     }
@@ -133,261 +108,668 @@ inline void ZBarrier::self_heal(volatile oop* p, uintptr_t addr, uintptr_t heal_
     // The oop location was healed by another barrier, but still needs upgrading.
     // Re-apply healing to make sure the oop is not left with weaker (remapped or
     // finalizable) metadata bits than what this barrier tried to apply.
-    assert(ZAddress::offset(prev_addr) == ZAddress::offset(heal_addr), "Invalid offset");
-    addr = prev_addr;
+    ptr = prev_ptr;
   }
 }
 
-template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path>
-inline oop ZBarrier::barrier(volatile oop* p, oop o) {
-  const uintptr_t addr = ZOop::to_address(o);
+inline ZGeneration* ZBarrier::remap_generation(zpointer ptr) {
+  assert(!ZPointer::is_load_good(ptr), "no need to remap load-good pointer");
+
+  if (ZPointer::is_old_load_good(ptr)) {
+    return ZGeneration::young();
+  }
+
+  if (ZPointer::is_young_load_good(ptr)) {
+    return ZGeneration::old();
+  }
+
+  // Double remap bad - the pointer is neither old load good nor
+  // young load good. First the code ...
+
+  const uintptr_t remembered_bits = untype(ptr) & ZPointerRememberedMask;
+  const bool old_to_old_ptr = remembered_bits == ZPointerRememberedMask;
+
+  if (old_to_old_ptr) {
+    return ZGeneration::old();
+  }
+
+  const zaddress_unsafe addr = ZPointer::uncolor_unsafe(ptr);
+  if (ZGeneration::young()->forwarding(addr) != nullptr) {
+    assert(ZGeneration::old()->forwarding(addr) == nullptr, "Mutually exclusive");
+    return ZGeneration::young();
+  } else {
+    return ZGeneration::old();
+  }
+
+  // ... then the explanation. Time to put your seat belt on.
+
+  // In this context we only have access to the ptr (colored oop), but we
+  // don't know if this refers to a stale young gen or old gen object.
+  // However, by being careful with when we run young and old collections,
+  // and by explicitly remapping roots we can figure this out by looking
+  // at the metadata bits in the pointer.
+
+  // *Roots (including remset)*:
+  //
+  // will never have double remap bit errors,
+  // and will never enter this path. The reason is that there's always a
+  // phase that remaps all roots between all relocation phases:
+  //
+  // 1) Young marking remaps the roots, before the young relocation runs
+  //
+  // 2) The old roots_remap phase blocks out young collections and runs just
+  //    before old relocation starts
+
+  // *Heap object fields*:
+  //
+  // could have double remap bit errors, and may enter this path. We are using
+  // knowledge about how *remember* bits are set, to narrow down the
+  // possibilities.
+
+  // Short summary:
+  //
+  // If both remember bits are set, when we have a double
+  // remap bit error, then we know that we are dealing with
+  // an old-to-old pointer.
+  //
+  // Otherwise, we are dealing with a young-to-any pointer,
+  // and the address that contained the pointed-to object, is
+  // guaranteed to have only been used by either the young gen
+  // or the old gen.
+
+  // Longer explanation:
+
+  // Double remap bad pointers in young gen:
+  //
+  // After young relocation, the young gen objects were promoted to old gen,
+  // and we keep track of those old-to-young pointers via the remset
+  // (described above in the roots section).
+  //
+  // However, when young marking started, the current set of young gen objects
+  // are snapshotted, and subsequent allocations end up in the next young
+  // collection. Between young mark start, and young relocate start, stores
+  // can happen to either the "young allocating" objects, or objects that
+  // are about to become survivors. For both survivors and young-allocating
+  // objects, it is true that their zpointers will be store good when
+  // young marking finishes, and can not get demoted. These pointers will become
+  // young remap bad after young relocate start. We don't maintain a remset
+  // for the young allocating objects, so we don't have the same guarantee as
+  // we have for roots (including remset). Pointers in these objects are
+  // therefore therefore susceptible to become double remap bad.
+  //
+  // The scenario that can happen is:
+  //   - Store in young allocating or future survivor happens between young mark
+  //     start and young relocate start
+  //   - Young relocate start makes this pointer young remap bad
+  //   - It is NOT fixed in roots_remap (it is not part of the remset or roots)
+  //   - Old relocate start makes this pointer also old remap bad
+
+  // Double remap bad pointers in old gen:
+  //
+  // When an object is promoted, all oop*s are added to the remset. (Could
+  // have either double or single remember bits at this point)
+  //
+  // As long as we have a remset entry for the oop*, we ensure that the pointer
+  // is not double remap bad. See the roots section.
+  //
+  // However, at some point the GC notices that the pointer points to an old
+  // object, and that there's no need for a remset entry. Because of that,
+  // the young collection will not visit the pointer, and the pointer can
+  // become double remap bad.
+  //
+  // The scenario that can happen is:
+  //   - Old marking visits the object
+  //   - Old relocation starts and then young relocation starts
+  //      or
+  //   - Young relocation starts and then old relocation starts
+
+  // About double *remember* bits:
+  //
+  // Whenever we:
+  // - perform a store barrier, we heal with one remember bit.
+  // - mark objects in young gen, we heal with one remember bit.
+  // - perform a non-store barrier outside of young gen, we heal with
+  //   double remember bits.
+  // - "remset forget" a pointer in an old object, we heal with double
+  //   remember bits.
+  //
+  // Double remember bits ensures that *every* store that encounters it takes
+  // a slow path.
+  //
+  // If we encounter a pointer that is both double remap bad *and* has double
+  // remember bits, we know that it can't be young and it has to be old!
+  //
+  // Pointers in young objects:
+  //
+  // The only double remap bad young pointers are inside "young allocating"
+  // objects and survivors, as described above. When such a pointer was written
+  // into the young allocating memory, or marked in young gen, the pointer was
+  // remap good and the store/young mark barrier healed with a single remember bit.
+  // No other barrier could replace that bit, because store good is the greatest
+  // barrier, and all other barriers will take the fast-path. This is true until
+  // the young relocation starts.
+  //
+  // After the young relocation has started, the pointer became young remap
+  // bad, and maybe we even started an old relocation, and the pointer became
+  // double remap bad. When the next load barrier triggers, it will self heal
+  // with double remember bits, but *importantly* it will at the same time
+  // heal with good remap bits.
+  //
+  // So, if we have entered this "double remap bad" path, and the pointer was
+  // located in young gen, then it was young allocating or a survivor, and it
+  // must only have one remember bit set!
+  //
+  // Pointers in old objects:
+  //
+  // When pointers become forgotten, they are tagged with double remembered
+  // bits. Only way to convert the pointer into having only one remembered
+  // bit, is to perform a store. When that happens, the pointer becomes both
+  // remap good and remembered again, and will be handled as the roots
+  // described above.
+
+  // With the above information:
+  //
+  // Iff we find a double remap bad pointer with *double remember bits*,
+  // then we know that it is an old-to-old pointer, and we should use the
+  // forwarding table of the old generation.
+  //
+  // Iff we find a double remap bad pointer with a *single remember bit*,
+  // then we know that it is a young-to-any pointer. We still don't know
+  // if the pointed-to object is young or old.
+
+  // Figuring out if a double remap bad pointer in young pointed at
+  // young or old:
+  //
+  // The scenario that created a double remap bad pointer in the young
+  // allocating or survivor memory is that it was written during the last
+  // young marking before the old relocation started. At that point, the old
+  // generation collection has already taken its marking snapshot, and
+  // determined what pages will be marked and therefore eligible to become
+  // part of the old relocation set. If the young generation relocated/freed
+  // a page (address range), and that address range was then reused for an old
+  // page, it won't be part of the old snapshot and it therefore won't be
+  // selected for old relocation.
+  //
+  // Because of this, we know that the object written into the young
+  // allocating page will at most belong to one of the two relocation sets,
+  // and we can therefore simply check in which table we installed
+  // ZForwarding.
+}
+
+inline zaddress ZBarrier::make_load_good(zpointer o) {
+  if (is_null_any(o)) {
+    return zaddress::null;
+  }
+
+  if (ZPointer::is_load_good_or_null(o)) {
+    return ZPointer::uncolor(o);
+  }
+
+  return relocate_or_remap(ZPointer::uncolor_unsafe(o), remap_generation(o));
+}
+
+inline zaddress ZBarrier::make_load_good_no_relocate(zpointer o) {
+  if (is_null_any(o)) {
+    return zaddress::null;
+  }
+
+  if (ZPointer::is_load_good_or_null(o)) {
+    return ZPointer::uncolor(o);
+  }
+
+  return remap(ZPointer::uncolor_unsafe(o), remap_generation(o));
+}
+
+inline void z_assert_is_barrier_safe() {
+  assert(!Thread::current()->is_ConcurrentGC_thread() ||          /* Need extra checks for ConcurrentGCThreads */
+         Thread::current()->is_suspendible_thread() ||            /* Thread prevents safepoints */
+         Thread::current()->is_indirectly_suspendible_thread() || /* Coordinator thread prevents safepoints */
+         SafepointSynchronize::is_at_safepoint(),                 /* Is at safepoint */
+         "Shouldn't perform load barrier");
+}
+
+template <typename ZBarrierSlowPath>
+inline zaddress ZBarrier::barrier(ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path, ZBarrierColor color, volatile zpointer* p, zpointer o, bool allow_null) {
+  z_assert_is_barrier_safe();
 
   // Fast path
-  if (fast_path(addr)) {
-    return ZOop::from_address(addr);
+  if (fast_path(o)) {
+    return ZPointer::uncolor(o);
   }
 
+  // Make load good
+  const zaddress load_good_addr = make_load_good(o);
+
   // Slow path
-  const uintptr_t good_addr = slow_path(addr);
+  const zaddress good_addr = slow_path(load_good_addr);
 
-  if (p != NULL) {
-    self_heal<fast_path>(p, addr, good_addr);
+  // Self heal
+  if (p != nullptr) {
+    // Color
+    const zpointer good_ptr = color(good_addr, o);
+
+    assert(!is_null(good_ptr), "Always block raw null");
+
+    self_heal(fast_path, p, o, good_ptr, allow_null);
   }
 
-  return ZOop::from_address(good_addr);
+  return good_addr;
 }
 
-template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path>
-inline oop ZBarrier::weak_barrier(volatile oop* p, oop o) {
-  const uintptr_t addr = ZOop::to_address(o);
+inline void ZBarrier::remap_young_relocated(volatile zpointer* p, zpointer o) {
+  assert(ZPointer::is_old_load_good(o), "Should be old load good");
+  assert(!ZPointer::is_young_load_good(o), "Should not be young load good");
 
-  // Fast path
-  if (fast_path(addr)) {
-    // Return the good address instead of the weak good address
-    // to ensure that the currently active heap view is used.
-    return ZOop::from_address(ZAddress::good_or_null(addr));
+  // Make load good
+  const zaddress load_good_addr = make_load_good_no_relocate(o);
+
+  // Color
+  const zpointer good_ptr = ZAddress::load_good(load_good_addr,  o);
+
+  assert(!is_null(good_ptr), "Always block raw null");
+
+  // Despite knowing good_ptr isn't null in this context, we use the
+  // load_good_or_null fast path, because it is faster.
+  self_heal(is_load_good_or_null_fast_path, p, o, good_ptr, false /* allow_null */);
+}
+
+inline zpointer ZBarrier::load_atomic(volatile zpointer* p) {
+  const zpointer ptr = Atomic::load(p);
+  assert_is_valid(ptr);
+  return ptr;
+}
+
+//
+// Fast paths
+//
+
+inline bool ZBarrier::is_load_good_or_null_fast_path(zpointer ptr) {
+  return ZPointer::is_load_good_or_null(ptr);
+}
+
+inline bool ZBarrier::is_mark_good_fast_path(zpointer ptr) {
+  return ZPointer::is_mark_good(ptr);
+}
+
+inline bool ZBarrier::is_store_good_fast_path(zpointer ptr) {
+  return ZPointer::is_store_good(ptr);
+}
+
+inline bool ZBarrier::is_store_good_or_null_fast_path(zpointer ptr) {
+  return ZPointer::is_store_good_or_null(ptr);
+}
+
+inline bool ZBarrier::is_store_good_or_null_any_fast_path(zpointer ptr) {
+  return is_null_any(ptr) || !ZPointer::is_store_bad(ptr);
+}
+
+inline bool ZBarrier::is_mark_young_good_fast_path(zpointer ptr) {
+  return ZPointer::is_load_good(ptr) && ZPointer::is_marked_young(ptr);
+}
+
+inline bool ZBarrier::is_finalizable_good_fast_path(zpointer ptr) {
+  return ZPointer::is_load_good(ptr) && ZPointer::is_marked_any_old(ptr);
+}
+
+//
+// Slow paths
+//
+
+inline zaddress ZBarrier::promote_slow_path(zaddress addr) {
+  // No need to do anything
+  return addr;
+}
+
+//
+// Color functions
+//
+
+inline zpointer color_load_good(zaddress new_addr, zpointer old_ptr) {
+  return ZAddress::load_good(new_addr, old_ptr);
+}
+
+inline zpointer color_finalizable_good(zaddress new_addr, zpointer old_ptr) {
+  if (ZPointer::is_marked_old(old_ptr)) {
+    // Don't down-grade pointers
+    return ZAddress::mark_old_good(new_addr, old_ptr);
+  } else {
+    return ZAddress::finalizable_good(new_addr, old_ptr);
   }
+}
 
-  // Slow path
-  const uintptr_t good_addr = slow_path(addr);
+inline zpointer color_mark_good(zaddress new_addr, zpointer old_ptr) {
+  return ZAddress::mark_good(new_addr, old_ptr);
+}
 
-  if (p != NULL) {
-    // The slow path returns a good/marked address or null, but we never mark
-    // oops in a weak load barrier so we always heal with the remapped address.
-    self_heal<fast_path>(p, addr, ZAddress::remapped_or_null(good_addr));
+inline zpointer color_mark_young_good(zaddress new_addr, zpointer old_ptr) {
+  return ZAddress::mark_young_good(new_addr, old_ptr);
+}
+
+inline zpointer color_remset_good(zaddress new_addr, zpointer old_ptr) {
+  if (new_addr == zaddress::null || ZHeap::heap()->is_young(new_addr)) {
+    return ZAddress::mark_good(new_addr, old_ptr);
+  } else {
+    // If remembered set scanning finds an old-to-old pointer, we won't mark it
+    // and hence only really care about setting remembered bits to 11 so that
+    // subsequent stores trip on the store-bad bit pattern. However, the contract
+    // with the fast path check, is that the pointer should invariantly be young
+    // mark good at least, so we color it as such.
+    return ZAddress::mark_young_good(new_addr, old_ptr);
   }
-
-  return ZOop::from_address(good_addr);
 }
 
-template <ZBarrierFastPath fast_path, ZBarrierSlowPath slow_path>
-inline void ZBarrier::root_barrier(oop* p, oop o) {
-  const uintptr_t addr = ZOop::to_address(o);
-
-  // Fast path
-  if (fast_path(addr)) {
-    return;
-  }
-
-  // Slow path
-  const uintptr_t good_addr = slow_path(addr);
-
-  // Non-atomic healing helps speed up root scanning. This is safe to do
-  // since we are always healing roots in a safepoint, or under a lock,
-  // which ensures we are never racing with mutators modifying roots while
-  // we are healing them. It's also safe in case multiple GC threads try
-  // to heal the same root if it is aligned, since they would always heal
-  // the root in the same way and it does not matter in which order it
-  // happens. For misaligned oops, there needs to be mutual exclusion.
-  *p = ZOop::from_address(good_addr);
-}
-
-inline bool ZBarrier::is_good_or_null_fast_path(uintptr_t addr) {
-  return ZAddress::is_good_or_null(addr);
-}
-
-inline bool ZBarrier::is_weak_good_or_null_fast_path(uintptr_t addr) {
-  return ZAddress::is_weak_good_or_null(addr);
-}
-
-inline bool ZBarrier::is_marked_or_null_fast_path(uintptr_t addr) {
-  return ZAddress::is_marked_or_null(addr);
-}
-
-inline bool ZBarrier::during_mark() {
-  return ZGlobalPhase == ZPhaseMark;
-}
-
-inline bool ZBarrier::during_relocate() {
-  return ZGlobalPhase == ZPhaseRelocate;
+inline zpointer color_store_good(zaddress new_addr, zpointer old_ptr) {
+  return ZAddress::store_good(new_addr);
 }
 
 //
 // Load barrier
 //
-inline oop ZBarrier::load_barrier_on_oop(oop o) {
-  return load_barrier_on_oop_field_preloaded((oop*)NULL, o);
-}
 
-inline oop ZBarrier::load_barrier_on_oop_field(volatile oop* p) {
-  const oop o = Atomic::load(p);
+inline zaddress ZBarrier::load_barrier_on_oop_field(volatile zpointer* p) {
+  const zpointer o = load_atomic(p);
   return load_barrier_on_oop_field_preloaded(p, o);
 }
 
-inline oop ZBarrier::load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) {
-  return barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o);
+inline zaddress ZBarrier::load_barrier_on_oop_field_preloaded(volatile zpointer* p, zpointer o) {
+  auto slow_path = [](zaddress addr) -> zaddress {
+    return addr;
+  };
+
+  return barrier(is_load_good_or_null_fast_path, slow_path, color_load_good, p, o);
 }
 
-inline void ZBarrier::load_barrier_on_oop_array(volatile oop* p, size_t length) {
-  for (volatile const oop* const end = p + length; p < end; p++) {
-    load_barrier_on_oop_field(p);
-  }
+inline zaddress ZBarrier::keep_alive_load_barrier_on_oop_field_preloaded(volatile zpointer* p, zpointer o) {
+  assert(!ZResurrection::is_blocked(), "This operation is only valid when resurrection is not blocked");
+  return barrier(is_mark_good_fast_path, keep_alive_slow_path, color_mark_good, p, o);
 }
 
-inline oop ZBarrier::load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) {
+//
+// Load barrier on non-strong oop refs
+//
+
+inline zaddress ZBarrier::load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o) {
   verify_on_weak(p);
 
   if (ZResurrection::is_blocked()) {
-    return barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o);
+    return blocking_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
   }
 
-  return load_barrier_on_oop_field_preloaded(p, o);
+  return keep_alive_load_barrier_on_oop_field_preloaded(p, o);
 }
 
-inline oop ZBarrier::load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) {
+inline zaddress ZBarrier::load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o) {
   if (ZResurrection::is_blocked()) {
-    return barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o);
+    return blocking_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
   }
 
-  return load_barrier_on_oop_field_preloaded(p, o);
+  return keep_alive_load_barrier_on_oop_field_preloaded(p, o);
 }
 
-inline void ZBarrier::load_barrier_on_root_oop_field(oop* p) {
-  const oop o = *p;
-  root_barrier<is_good_or_null_fast_path, load_barrier_on_oop_slow_path>(p, o);
-}
-
-inline void ZBarrier::load_barrier_on_invisible_root_oop_field(oop* p) {
-  const oop o = *p;
-  root_barrier<is_good_or_null_fast_path, load_barrier_on_invisible_root_oop_slow_path>(p, o);
-}
-
-//
-// Weak load barrier
-//
-inline oop ZBarrier::weak_load_barrier_on_oop_field(volatile oop* p) {
-  assert(!ZResurrection::is_blocked(), "Should not be called during resurrection blocked phase");
-  const oop o = Atomic::load(p);
-  return weak_load_barrier_on_oop_field_preloaded(p, o);
-}
-
-inline oop ZBarrier::weak_load_barrier_on_oop_field_preloaded(volatile oop* p, oop o) {
-  return weak_barrier<is_weak_good_or_null_fast_path, weak_load_barrier_on_oop_slow_path>(p, o);
-}
-
-inline oop ZBarrier::weak_load_barrier_on_weak_oop(oop o) {
-  return weak_load_barrier_on_weak_oop_field_preloaded((oop*)NULL, o);
-}
-
-inline oop ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(volatile oop* p, oop o) {
+inline zaddress ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o) {
   verify_on_weak(p);
 
   if (ZResurrection::is_blocked()) {
-    return barrier<is_good_or_null_fast_path, weak_load_barrier_on_weak_oop_slow_path>(p, o);
+    return blocking_load_barrier_on_weak_oop_field_preloaded(p, o);
   }
 
-  return weak_load_barrier_on_oop_field_preloaded(p, o);
+  // Normal load barrier doesn't keep the object alive
+  return load_barrier_on_oop_field_preloaded(p, o);
 }
 
-inline oop ZBarrier::weak_load_barrier_on_phantom_oop(oop o) {
-  return weak_load_barrier_on_phantom_oop_field_preloaded((oop*)NULL, o);
-}
-
-inline oop ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(volatile oop* p, oop o) {
+inline zaddress ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o) {
   if (ZResurrection::is_blocked()) {
-    return barrier<is_good_or_null_fast_path, weak_load_barrier_on_phantom_oop_slow_path>(p, o);
+    return blocking_load_barrier_on_phantom_oop_field_preloaded(p, o);
   }
 
-  return weak_load_barrier_on_oop_field_preloaded(p, o);
+  // Normal load barrier doesn't keep the object alive
+  return load_barrier_on_oop_field_preloaded(p, o);
+}
+
+inline zaddress ZBarrier::blocking_keep_alive_load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o) {
+  auto slow_path = [=](zaddress addr) -> zaddress {
+    return ZBarrier::blocking_keep_alive_on_weak_slow_path(p, addr);
+  };
+  return barrier(is_mark_good_fast_path, slow_path, color_mark_good, p, o);
+}
+
+inline zaddress ZBarrier::blocking_keep_alive_load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o) {
+  auto slow_path = [=](zaddress addr) -> zaddress {
+    return ZBarrier::blocking_keep_alive_on_phantom_slow_path(p, addr);
+  };
+  return barrier(is_mark_good_fast_path, slow_path, color_mark_good, p, o);
+}
+
+inline zaddress ZBarrier::blocking_load_barrier_on_weak_oop_field_preloaded(volatile zpointer* p, zpointer o) {
+  auto slow_path = [=](zaddress addr) -> zaddress {
+    return ZBarrier::blocking_load_barrier_on_weak_slow_path(p, addr);
+  };
+  return barrier(is_mark_good_fast_path, slow_path, color_mark_good, p, o);
+}
+
+inline zaddress ZBarrier::blocking_load_barrier_on_phantom_oop_field_preloaded(volatile zpointer* p, zpointer o) {
+  auto slow_path = [=](zaddress addr) -> zaddress {
+    return ZBarrier::blocking_load_barrier_on_phantom_slow_path(p, addr);
+  };
+  return barrier(is_mark_good_fast_path, slow_path, color_mark_good, p, o);
 }
 
 //
-// Is alive barrier
+// Clean barrier
 //
-inline bool ZBarrier::is_alive_barrier_on_weak_oop(oop o) {
-  // Check if oop is logically non-null. This operation
-  // is only valid when resurrection is blocked.
-  assert(ZResurrection::is_blocked(), "Invalid phase");
-  return weak_load_barrier_on_weak_oop(o) != NULL;
-}
 
-inline bool ZBarrier::is_alive_barrier_on_phantom_oop(oop o) {
-  // Check if oop is logically non-null. This operation
-  // is only valid when resurrection is blocked.
-  assert(ZResurrection::is_blocked(), "Invalid phase");
-  return weak_load_barrier_on_phantom_oop(o) != NULL;
-}
-
-//
-// Keep alive barrier
-//
-inline void ZBarrier::keep_alive_barrier_on_weak_oop_field(volatile oop* p) {
+inline bool ZBarrier::clean_barrier_on_weak_oop_field(volatile zpointer* p) {
   assert(ZResurrection::is_blocked(), "This operation is only valid when resurrection is blocked");
-  const oop o = Atomic::load(p);
-  barrier<is_good_or_null_fast_path, keep_alive_barrier_on_weak_oop_slow_path>(p, o);
+  const zpointer o = load_atomic(p);
+  auto slow_path = [=](zaddress addr) -> zaddress {
+    return ZBarrier::blocking_load_barrier_on_weak_slow_path(p, addr);
+  };
+  return is_null(barrier(is_mark_good_fast_path, slow_path, color_mark_good, p, o, true /* allow_null */));
 }
 
-inline void ZBarrier::keep_alive_barrier_on_phantom_oop_field(volatile oop* p) {
+inline bool ZBarrier::clean_barrier_on_phantom_oop_field(volatile zpointer* p) {
   assert(ZResurrection::is_blocked(), "This operation is only valid when resurrection is blocked");
-  const oop o = Atomic::load(p);
-  barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o);
+  const zpointer o = load_atomic(p);
+  auto slow_path = [=](zaddress addr) -> zaddress {
+    return ZBarrier::blocking_load_barrier_on_phantom_slow_path(p, addr);
+  };
+  return is_null(barrier(is_mark_good_fast_path, slow_path, color_mark_good, p, o, true /* allow_null */));
 }
 
-inline void ZBarrier::keep_alive_barrier_on_phantom_root_oop_field(oop* p) {
-  // The keep alive operation is only valid when resurrection is blocked.
-  //
-  // Except with Loom, where we intentionally trigger arms nmethods after
-  // unlinking, to get a sense of what nmethods are alive. This will trigger
-  // the keep alive barriers, but the oops are healed and the slow-paths
-  // will not trigger. We have stronger checks in the slow-paths.
-  assert(ZResurrection::is_blocked() || (CodeCache::contains((void*)p)),
-         "This operation is only valid when resurrection is blocked");
-  const oop o = *p;
-  root_barrier<is_good_or_null_fast_path, keep_alive_barrier_on_phantom_oop_slow_path>(p, o);
-}
+inline bool ZBarrier::clean_barrier_on_final_oop_field(volatile zpointer* p) {
+  assert(ZResurrection::is_blocked(), "Invalid phase");
 
-inline void ZBarrier::keep_alive_barrier_on_oop(oop o) {
-  const uintptr_t addr = ZOop::to_address(o);
-  assert(ZAddress::is_good(addr), "Invalid address");
+  // The referent in a FinalReference should never be cleared by the GC. Instead
+  // it should just be healed (as if it was a phantom oop) and this function should
+  // return true if the object pointer to by the referent is not strongly reachable.
+  const zpointer o = load_atomic(p);
+  auto slow_path = [=](zaddress addr) -> zaddress {
+    return ZBarrier::blocking_load_barrier_on_phantom_slow_path(p, addr);
+  };
+  const zaddress addr = barrier(is_mark_good_fast_path, slow_path, color_mark_good, p, o);
+  assert(!is_null(addr), "Should be finalizable marked");
 
-  if (during_mark()) {
-    keep_alive_barrier_on_oop_slow_path(addr);
-  }
+  return is_null(blocking_load_barrier_on_weak_slow_path(p, addr));
 }
 
 //
 // Mark barrier
 //
-inline void ZBarrier::mark_barrier_on_oop_field(volatile oop* p, bool finalizable) {
-  const oop o = Atomic::load(p);
+inline void ZBarrier::mark_barrier_on_oop_field(volatile zpointer* p, bool finalizable) {
+  const zpointer o = load_atomic(p);
 
   if (finalizable) {
-    barrier<is_marked_or_null_fast_path, mark_barrier_on_finalizable_oop_slow_path>(p, o);
+    // During marking, we mark through already marked oops to avoid having
+    // some large part of the object graph hidden behind a pushed, but not
+    // yet flushed, entry on a mutator mark stack. Always marking through
+    // allows the GC workers to proceed through the object graph even if a
+    // mutator touched an oop first, which in turn will reduce the risk of
+    // having to flush mark stacks multiple times to terminate marking.
+    //
+    // However, when doing finalizable marking we don't always want to mark
+    // through. First, marking through an already strongly marked oop would
+    // be wasteful, since we will then proceed to do finalizable marking on
+    // an object which is, or will be, marked strongly. Second, marking
+    // through an already finalizable marked oop would also be wasteful,
+    // since such oops can never end up on a mutator mark stack and can
+    // therefore not hide some part of the object graph from GC workers.
+
+    // Make the oop finalizable marked/good, instead of normal marked/good.
+    // This is needed because an object might first becomes finalizable
+    // marked by the GC, and then loaded by a mutator thread. In this case,
+    // the mutator thread must be able to tell that the object needs to be
+    // strongly marked. The finalizable bit in the oop exists to make sure
+    // that a load of a finalizable marked oop will fall into the barrier
+    // slow path so that we can mark the object as strongly reachable.
+
+    // Note: that this does not color the pointer finalizable marked if it
+    // is already colored marked old good.
+    barrier(is_finalizable_good_fast_path, mark_finalizable_slow_path, color_finalizable_good, p, o);
   } else {
-    const uintptr_t addr = ZOop::to_address(o);
-    if (ZAddress::is_good(addr)) {
-      // Mark through good oop
-      mark_barrier_on_oop_slow_path(addr);
-    } else {
-      // Mark through bad oop
-      barrier<is_good_or_null_fast_path, mark_barrier_on_oop_slow_path>(p, o);
-    }
+    barrier(is_mark_good_fast_path, mark_slow_path, color_mark_good, p, o);
   }
 }
 
-inline void ZBarrier::mark_barrier_on_oop_array(volatile oop* p, size_t length, bool finalizable) {
-  for (volatile const oop* const end = p + length; p < end; p++) {
-    mark_barrier_on_oop_field(p, finalizable);
+inline void ZBarrier::mark_barrier_on_old_oop_field(volatile zpointer* p, bool finalizable) {
+  assert(ZHeap::heap()->is_old(p), "Should be from old");
+  const zpointer o = load_atomic(p);
+
+  if (finalizable) {
+    // During marking, we mark through already marked oops to avoid having
+    // some large part of the object graph hidden behind a pushed, but not
+    // yet flushed, entry on a mutator mark stack. Always marking through
+    // allows the GC workers to proceed through the object graph even if a
+    // mutator touched an oop first, which in turn will reduce the risk of
+    // having to flush mark stacks multiple times to terminate marking.
+    //
+    // However, when doing finalizable marking we don't always want to mark
+    // through. First, marking through an already strongly marked oop would
+    // be wasteful, since we will then proceed to do finalizable marking on
+    // an object which is, or will be, marked strongly. Second, marking
+    // through an already finalizable marked oop would also be wasteful,
+    // since such oops can never end up on a mutator mark stack and can
+    // therefore not hide some part of the object graph from GC workers.
+
+    // Make the oop finalizable marked/good, instead of normal marked/good.
+    // This is needed because an object might first becomes finalizable
+    // marked by the GC, and then loaded by a mutator thread. In this case,
+    // the mutator thread must be able to tell that the object needs to be
+    // strongly marked. The finalizable bit in the oop exists to make sure
+    // that a load of a finalizable marked oop will fall into the barrier
+    // slow path so that we can mark the object as strongly reachable.
+
+    // Note: that this does not color the pointer finalizable marked if it
+    // is already colored marked old good.
+    barrier(is_finalizable_good_fast_path, mark_finalizable_from_old_slow_path, color_finalizable_good, p, o);
+  } else {
+    barrier(is_mark_good_fast_path, mark_from_old_slow_path, color_mark_good, p, o);
+  }
+}
+
+inline void ZBarrier::mark_barrier_on_young_oop_field(volatile zpointer* p) {
+  assert(ZHeap::heap()->is_young(p), "Should be from young");
+  const zpointer o = load_atomic(p);
+  barrier(is_store_good_or_null_any_fast_path, mark_from_young_slow_path, color_store_good, p, o);
+}
+
+inline void ZBarrier::promote_barrier_on_young_oop_field(volatile zpointer* p) {
+  const zpointer o = load_atomic(p);
+  // Objects that get promoted to the old generation, must invariantly contain
+  // only store good pointers. However, the young marking code above filters
+  // out null pointers, so we need to explicitly ensure even null pointers are
+  // store good, before objects may get promoted (and before relocate start).
+  // This barrier ensures that.
+  // This could simply be ensured in the marking above, but promotion rates
+  // are typically rather low, and fixing all null pointers strictly, when
+  // only a few had to be store good due to promotions, is generally not favourable
+  barrier(is_store_good_fast_path, promote_slow_path, color_store_good, p, o);
+}
+
+inline zaddress ZBarrier::remset_barrier_on_oop_field(volatile zpointer* p) {
+  const zpointer o = load_atomic(p);
+  return barrier(is_mark_young_good_fast_path, mark_young_slow_path, color_remset_good, p, o);
+}
+
+inline void ZBarrier::mark_young_good_barrier_on_oop_field(volatile zpointer* p) {
+  const zpointer o = load_atomic(p);
+  barrier(is_mark_young_good_fast_path, mark_young_slow_path, color_mark_young_good, p, o);
+}
+
+//
+// Store barrier
+//
+
+inline void ZBarrier::store_barrier_on_heap_oop_field(volatile zpointer* p, bool heal) {
+  const zpointer prev = load_atomic(p);
+
+  auto slow_path = [=](zaddress addr) -> zaddress {
+    return ZBarrier::heap_store_slow_path(p, addr, prev, heal);
+  };
+
+  if (heal) {
+    barrier(is_store_good_fast_path, slow_path, color_store_good, p, prev);
+  } else {
+    barrier(is_store_good_or_null_fast_path, slow_path, color_store_good, nullptr, prev);
+  }
+}
+
+inline void ZBarrier::store_barrier_on_native_oop_field(volatile zpointer* p, bool heal) {
+  const zpointer prev = load_atomic(p);
+
+  if (heal) {
+    barrier(is_store_good_fast_path, native_store_slow_path, color_store_good, p, prev);
+  } else {
+    barrier(is_store_good_or_null_fast_path, native_store_slow_path, color_store_good, nullptr, prev);
+  }
+}
+
+inline void ZBarrier::no_keep_alive_store_barrier_on_heap_oop_field(volatile zpointer* p) {
+  const zpointer prev = load_atomic(p);
+
+  auto slow_path = [=](zaddress addr) -> zaddress {
+    return ZBarrier::no_keep_alive_heap_store_slow_path(p, addr);
+  };
+
+  barrier(is_store_good_fast_path, slow_path, color_store_good, nullptr, prev);
+}
+
+inline void ZBarrier::remember(volatile zpointer* p) {
+  if (ZHeap::heap()->is_old(p)) {
+    ZGeneration::young()->remember(p);
+  }
+}
+
+inline void ZBarrier::mark_and_remember(volatile zpointer* p, zaddress addr) {
+  if (!is_null(addr)) {
+    mark<ZMark::DontResurrect, ZMark::AnyThread, ZMark::Follow, ZMark::Strong>(addr);
+  }
+  remember(p);
+}
+
+template <bool resurrect, bool gc_thread, bool follow, bool finalizable>
+inline void ZBarrier::mark(zaddress addr) {
+  assert(!ZVerifyOops || oopDesc::is_oop(to_oop(addr), false), "must be oop");
+
+  if (ZHeap::heap()->is_old(addr)) {
+    ZGeneration::old()->mark_object_if_active<resurrect, gc_thread, follow, finalizable>(addr);
+  } else {
+    ZGeneration::young()->mark_object_if_active<resurrect, gc_thread, follow, ZMark::Strong>(addr);
+  }
+}
+
+template <bool resurrect, bool gc_thread, bool follow>
+inline void ZBarrier::mark_young(zaddress addr) {
+  assert(ZGeneration::young()->is_phase_mark(), "Should only be called during marking");
+  assert(!ZVerifyOops || oopDesc::is_oop(to_oop(addr), false), "must be oop");
+  assert(ZHeap::heap()->is_young(addr), "Must be young");
+
+  ZGeneration::young()->mark_object<resurrect, gc_thread, follow, ZMark::Strong>(addr);
+}
+
+template <bool resurrect, bool gc_thread, bool follow>
+inline void ZBarrier::mark_if_young(zaddress addr) {
+  if (ZHeap::heap()->is_young(addr)) {
+    mark_young<resurrect, gc_thread, follow>(addr);
   }
 }
 
diff --git a/src/hotspot/share/gc/z/zBarrierSet.cpp b/src/hotspot/share/gc/z/zBarrierSet.cpp
index d37623b8624..c2922f54fcc 100644
--- a/src/hotspot/share/gc/z/zBarrierSet.cpp
+++ b/src/hotspot/share/gc/z/zBarrierSet.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,11 +26,16 @@
 #include "gc/z/zBarrierSetAssembler.hpp"
 #include "gc/z/zBarrierSetNMethod.hpp"
 #include "gc/z/zBarrierSetStackChunk.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zStackWatermark.hpp"
 #include "gc/z/zThreadLocalData.hpp"
+#include "runtime/deoptimization.hpp"
+#include "runtime/frame.inline.hpp"
 #include "runtime/javaThread.hpp"
+#include "runtime/registerMap.hpp"
+#include "runtime/stackWatermarkSet.hpp"
 #include "utilities/macros.hpp"
 #ifdef COMPILER1
 #include "gc/z/c1/zBarrierSetC1.hpp"
@@ -80,12 +85,18 @@ void ZBarrierSet::on_thread_destroy(Thread* thread) {
 }
 
 void ZBarrierSet::on_thread_attach(Thread* thread) {
-  // Set thread local address bad mask
-  ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask);
+  // Set thread local masks
+  ZThreadLocalData::set_load_bad_mask(thread, ZPointerLoadBadMask);
+  ZThreadLocalData::set_load_good_mask(thread, ZPointerLoadGoodMask);
+  ZThreadLocalData::set_mark_bad_mask(thread, ZPointerMarkBadMask);
+  ZThreadLocalData::set_store_bad_mask(thread, ZPointerStoreBadMask);
+  ZThreadLocalData::set_store_good_mask(thread, ZPointerStoreGoodMask);
+  ZThreadLocalData::set_nmethod_disarmed(thread, ZPointerStoreGoodMask);
   if (thread->is_Java_thread()) {
     JavaThread* const jt = JavaThread::cast(thread);
     StackWatermark* const watermark = new ZStackWatermark(jt);
     StackWatermarkSet::add_watermark(jt, watermark);
+    ZThreadLocalData::store_barrier_buffer(jt)->initialize();
   }
 }
 
@@ -94,6 +105,53 @@ void ZBarrierSet::on_thread_detach(Thread* thread) {
   ZHeap::heap()->mark_flush_and_free(thread);
 }
 
+static void deoptimize_allocation(JavaThread* thread) {
+  RegisterMap reg_map(thread, RegisterMap::UpdateMap::skip,
+                      RegisterMap::ProcessFrames::include,
+                      RegisterMap::WalkContinuation::skip);
+  const frame runtime_frame = thread->last_frame();
+  assert(runtime_frame.is_runtime_frame(), "must be runtime frame");
+
+  const frame caller_frame = runtime_frame.sender(&reg_map);
+  assert(caller_frame.is_compiled_frame(), "must be compiled");
+
+  const nmethod* const nm = caller_frame.cb()->as_nmethod();
+  if (nm->is_compiled_by_c2() && !caller_frame.is_deoptimized_frame()) {
+    Deoptimization::deoptimize_frame(thread, caller_frame.id());
+  }
+}
+
+void ZBarrierSet::on_slowpath_allocation_exit(JavaThread* thread, oop new_obj) {
+  const ZPage* const page = ZHeap::heap()->page(to_zaddress(new_obj));
+  const ZPageAge age = page->age();
+  if (age == ZPageAge::old) {
+    // We promised C2 that its allocations would end up in young gen. This object
+    // breaks that promise. Take a few steps in the interpreter instead, which has
+    // no such assumptions about where an object resides.
+    deoptimize_allocation(thread);
+    return;
+  }
+
+  if (!ZGeneration::young()->is_phase_mark_complete()) {
+    return;
+  }
+
+  if (!page->is_relocatable()) {
+    return;
+  }
+
+  if (ZRelocate::compute_to_age(age) != ZPageAge::old) {
+    return;
+  }
+
+  // If the object is young, we have to still be careful that it isn't racingly
+  // about to get promoted to the old generation. That causes issues when null
+  // pointers are supposed to be coloured, but the JIT is a bit sloppy and
+  // reinitializes memory with raw nulls. We detect this situation and detune
+  // rather than relying on the JIT to never be sloppy with redundant initialization.
+  deoptimize_allocation(thread);
+}
+
 void ZBarrierSet::print_on(outputStream* st) const {
   st->print_cr("ZBarrierSet");
 }
diff --git a/src/hotspot/share/gc/z/zBarrierSet.hpp b/src/hotspot/share/gc/z/zBarrierSet.hpp
index ebb80e106af..213f85dcea8 100644
--- a/src/hotspot/share/gc/z/zBarrierSet.hpp
+++ b/src/hotspot/share/gc/z/zBarrierSet.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,10 +25,14 @@
 #define SHARE_GC_Z_ZBARRIERSET_HPP
 
 #include "gc/shared/barrierSet.hpp"
+#include "gc/z/zAddress.hpp"
 
 class ZBarrierSetAssembler;
 
 class ZBarrierSet : public BarrierSet {
+private:
+  static zpointer store_good(oop obj);
+
 public:
   ZBarrierSet();
 
@@ -40,6 +44,8 @@ public:
   virtual void on_thread_attach(Thread* thread);
   virtual void on_thread_detach(Thread* thread);
 
+  virtual void on_slowpath_allocation_exit(JavaThread* thread, oop new_obj);
+
   virtual void print_on(outputStream* st) const;
 
   template <DecoratorSet decorators, typename BarrierSetT = ZBarrierSet>
@@ -53,48 +59,96 @@ public:
     template <DecoratorSet expected>
     static void verify_decorators_absent();
 
-    static oop* field_addr(oop base, ptrdiff_t offset);
+    static zpointer* field_addr(oop base, ptrdiff_t offset);
 
-    template <typename T>
-    static oop load_barrier_on_oop_field_preloaded(T* addr, oop o);
+    static zaddress load_barrier(zpointer* p, zpointer o);
+    static zaddress load_barrier_on_unknown_oop_ref(oop base, ptrdiff_t offset, zpointer* p, zpointer o);
 
-    template <typename T>
-    static oop load_barrier_on_unknown_oop_field_preloaded(oop base, ptrdiff_t offset, T* addr, oop o);
+    static void store_barrier_heap_with_healing(zpointer* p);
+    static void store_barrier_heap_without_healing(zpointer* p);
+    static void no_keep_alive_store_barrier_heap(zpointer* p);
+
+    static void store_barrier_native_with_healing(zpointer* p);
+    static void store_barrier_native_without_healing(zpointer* p);
+
+    static void unsupported();
+    static zaddress load_barrier(narrowOop* p, zpointer o) { unsupported(); return zaddress::null; }
+    static zaddress load_barrier_on_unknown_oop_ref(oop base, ptrdiff_t offset, narrowOop* p, zpointer o) { unsupported(); return zaddress::null; }
+    static void store_barrier_heap_with_healing(narrowOop* p) { unsupported(); }
+    static void store_barrier_heap_without_healing(narrowOop* p)  { unsupported(); }
+    static void no_keep_alive_store_barrier_heap(narrowOop* p)  { unsupported(); }
+    static void store_barrier_native_with_healing(narrowOop* p)  { unsupported(); }
+    static void store_barrier_native_without_healing(narrowOop* p)  { unsupported(); }
+
+    static zaddress oop_copy_one_barriers(zpointer* dst, zpointer* src);
+    static bool oop_copy_one_check_cast(zpointer* dst, zpointer* src, Klass* dst_klass);
+    static void oop_copy_one(zpointer* dst, zpointer* src);
+
+    static bool oop_arraycopy_in_heap_check_cast(zpointer* dst, zpointer* src, size_t length, Klass* dst_klass);
+    static bool oop_arraycopy_in_heap_no_check_cast(zpointer* dst, zpointer* src, size_t length);
 
   public:
     //
     // In heap
     //
-    template <typename T>
-    static oop oop_load_in_heap(T* addr);
+    static oop oop_load_in_heap(zpointer* p);
+    static oop oop_load_in_heap(oop* p)       { return oop_load_in_heap((zpointer*)p); };
+    static oop oop_load_in_heap(narrowOop* p) { unsupported(); return nullptr; }
+
     static oop oop_load_in_heap_at(oop base, ptrdiff_t offset);
 
-    template <typename T>
-    static oop oop_atomic_cmpxchg_in_heap(T* addr, oop compare_value, oop new_value);
+    static void oop_store_in_heap(zpointer* p, oop value);
+    static void oop_store_in_heap(oop* p, oop value)       { oop_store_in_heap((zpointer*)p, value); }
+    static void oop_store_in_heap(narrowOop* p, oop value) { unsupported(); }
+    static void oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value);
+
+    static void oop_store_not_in_heap(zpointer* p, oop value);
+    static void oop_store_not_in_heap(oop* p, oop value)       { oop_store_not_in_heap((zpointer*)p, value); }
+    static void oop_store_not_in_heap(narrowOop* p, oop value) { unsupported(); }
+    static void oop_store_not_in_heap_at(oop base, ptrdiff_t offset, oop value);
+
+    static oop oop_atomic_cmpxchg_in_heap(zpointer* p, oop compare_value, oop new_value);
+    static oop oop_atomic_cmpxchg_in_heap(oop* p, oop compare_value, oop new_value)       { return oop_atomic_cmpxchg_in_heap((zpointer*)p, compare_value, new_value); }
+    static oop oop_atomic_cmpxchg_in_heap(narrowOop* p, oop compare_value, oop new_value) { unsupported(); return nullptr; }
     static oop oop_atomic_cmpxchg_in_heap_at(oop base, ptrdiff_t offset, oop compare_value, oop new_value);
 
-    template <typename T>
-    static oop oop_atomic_xchg_in_heap(T* addr, oop new_value);
+    static oop oop_atomic_xchg_in_heap(zpointer* p, oop new_value);
+    static oop oop_atomic_xchg_in_heap(oop* p, oop new_value)       { return oop_atomic_xchg_in_heap((zpointer*)p, new_value); }
+    static oop oop_atomic_xchg_in_heap(narrowOop* p, oop new_value) { unsupported(); return nullptr; }
     static oop oop_atomic_xchg_in_heap_at(oop base, ptrdiff_t offset, oop new_value);
 
-    template <typename T>
-    static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
-                                      arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
+    static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, zpointer* src_raw,
+                                      arrayOop dst_obj, size_t dst_offset_in_bytes, zpointer* dst_raw,
                                       size_t length);
+    static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, oop* src_raw,
+                                      arrayOop dst_obj, size_t dst_offset_in_bytes, oop* dst_raw,
+                                      size_t length) {
+      return oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, (zpointer*)src_raw,
+                                   dst_obj, dst_offset_in_bytes, (zpointer*)dst_raw,
+                                   length);
+    }
+    static bool oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, narrowOop* src_raw,
+                                      arrayOop dst_obj, size_t dst_offset_in_bytes, narrowOop* dst_raw,
+                                      size_t length) { unsupported(); return false; }
 
     static void clone_in_heap(oop src, oop dst, size_t size);
 
     //
     // Not in heap
     //
-    template <typename T>
-    static oop oop_load_not_in_heap(T* addr);
+    static oop oop_load_not_in_heap(zpointer* p);
+    static oop oop_load_not_in_heap(oop* p);
+    static oop oop_load_not_in_heap(narrowOop* p) { unsupported(); return nullptr; }
 
-    template <typename T>
-    static oop oop_atomic_cmpxchg_not_in_heap(T* addr, oop compare_value, oop new_value);
+    static oop oop_atomic_cmpxchg_not_in_heap(zpointer* p, oop compare_value, oop new_value);
+    static oop oop_atomic_cmpxchg_not_in_heap(oop* p, oop compare_value, oop new_value) {
+      return oop_atomic_cmpxchg_not_in_heap((zpointer*)p, compare_value, new_value);
+    }
+    static oop oop_atomic_cmpxchg_not_in_heap(narrowOop* addr, oop compare_value, oop new_value) { unsupported(); return nullptr; }
 
-    template <typename T>
-    static oop oop_atomic_xchg_not_in_heap(T* addr, oop new_value);
+    static oop oop_atomic_xchg_not_in_heap(zpointer* p, oop new_value);
+    static oop oop_atomic_xchg_not_in_heap(oop* p, oop new_value)       { return oop_atomic_xchg_not_in_heap((zpointer*)p, new_value); }
+    static oop oop_atomic_xchg_not_in_heap(narrowOop* p, oop new_value) { unsupported(); return nullptr; }
   };
 };
 
diff --git a/src/hotspot/share/gc/z/zBarrierSet.inline.hpp b/src/hotspot/share/gc/z/zBarrierSet.inline.hpp
index 0970b519e10..bfbae74972d 100644
--- a/src/hotspot/share/gc/z/zBarrierSet.inline.hpp
+++ b/src/hotspot/share/gc/z/zBarrierSet.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,7 +27,11 @@
 #include "gc/z/zBarrierSet.hpp"
 
 #include "gc/shared/accessBarrierSupport.inline.hpp"
+#include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zBarrier.inline.hpp"
+#include "gc/z/zIterator.inline.hpp"
+#include "gc/z/zNMethod.hpp"
+#include "memory/iterator.inline.hpp"
 #include "utilities/debug.hpp"
 
 template <DecoratorSet decorators, typename BarrierSetT>
@@ -47,40 +51,44 @@ inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::verify_decorato
 }
 
 template <DecoratorSet decorators, typename BarrierSetT>
-inline oop* ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::field_addr(oop base, ptrdiff_t offset) {
-  assert(base != NULL, "Invalid base");
-  return reinterpret_cast<oop*>(reinterpret_cast<intptr_t>((void*)base) + offset);
+inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::unsupported() {
+  ShouldNotReachHere();
 }
 
 template <DecoratorSet decorators, typename BarrierSetT>
-template <typename T>
-inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier_on_oop_field_preloaded(T* addr, oop o) {
+inline zpointer* ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::field_addr(oop base, ptrdiff_t offset) {
+  assert(base != nullptr, "Invalid base");
+  return reinterpret_cast<zpointer*>(reinterpret_cast<intptr_t>((void*)base) + offset);
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier(zpointer* p, zpointer o) {
   verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
 
   if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
     if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
-      return ZBarrier::weak_load_barrier_on_oop_field_preloaded(addr, o);
+      // Load barriers on strong oop refs don't keep objects alive
+      return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
     } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
-      return ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(addr, o);
+      return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
     } else {
       assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
-      return ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(addr, o);
+      return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
     }
   } else {
     if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
-      return ZBarrier::load_barrier_on_oop_field_preloaded(addr, o);
+      return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
     } else if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
-      return ZBarrier::load_barrier_on_weak_oop_field_preloaded(addr, o);
+      return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
     } else {
       assert((HasDecorator<decorators, ON_PHANTOM_OOP_REF>::value), "Must be");
-      return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(addr, o);
+      return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
     }
   }
 }
 
 template <DecoratorSet decorators, typename BarrierSetT>
-template <typename T>
-inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier_on_unknown_oop_field_preloaded(oop base, ptrdiff_t offset, T* addr, oop o) {
+inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier_on_unknown_oop_ref(oop base, ptrdiff_t offset, zpointer* p, zpointer o) {
   verify_decorators_present<ON_UNKNOWN_OOP_REF>();
 
   const DecoratorSet decorators_known_strength =
@@ -88,57 +96,182 @@ inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::load_barrier_on_
 
   if (HasDecorator<decorators, AS_NO_KEEPALIVE>::value) {
     if (decorators_known_strength & ON_STRONG_OOP_REF) {
-      return ZBarrier::weak_load_barrier_on_oop_field_preloaded(addr, o);
+      // Load barriers on strong oop refs don't keep objects alive
+      return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
     } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
-      return ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(addr, o);
+      return ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded(p, o);
     } else {
       assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
-      return ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(addr, o);
+      return ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded(p, o);
     }
   } else {
     if (decorators_known_strength & ON_STRONG_OOP_REF) {
-      return ZBarrier::load_barrier_on_oop_field_preloaded(addr, o);
+      return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
     } else if (decorators_known_strength & ON_WEAK_OOP_REF) {
-      return ZBarrier::load_barrier_on_weak_oop_field_preloaded(addr, o);
+      return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
     } else {
       assert(decorators_known_strength & ON_PHANTOM_OOP_REF, "Must be");
-      return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(addr, o);
+      return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
     }
   }
 }
 
+inline zpointer ZBarrierSet::store_good(oop obj) {
+  assert(ZPointerStoreGoodMask != 0, "sanity");
+
+  const zaddress addr = to_zaddress(obj);
+  return ZAddress::store_good(addr);
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_with_healing(zpointer* p) {
+  if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
+    ZBarrier::store_barrier_on_heap_oop_field(p, true /* heal */);
+  } else {
+    assert(false, "Should not be used on uninitialized memory");
+  }
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_heap_without_healing(zpointer* p) {
+  if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
+    ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */);
+  }
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::no_keep_alive_store_barrier_heap(zpointer* p) {
+  if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
+    ZBarrier::no_keep_alive_store_barrier_on_heap_oop_field(p);
+  }
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_with_healing(zpointer* p) {
+  if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
+    ZBarrier::store_barrier_on_native_oop_field(p, true /* heal */);
+  } else {
+    assert(false, "Should not be used on uninitialized memory");
+  }
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::store_barrier_native_without_healing(zpointer* p) {
+  if (!HasDecorator<decorators, IS_DEST_UNINITIALIZED>::value) {
+    ZBarrier::store_barrier_on_native_oop_field(p, false /* heal */);
+  }
+}
+
 //
 // In heap
 //
 template <DecoratorSet decorators, typename BarrierSetT>
-template <typename T>
-inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(T* addr) {
+inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap(zpointer* p) {
   verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
 
-  const oop o = Raw::oop_load_in_heap(addr);
-  return load_barrier_on_oop_field_preloaded(addr, o);
+  const zpointer o = Raw::load_in_heap(p);
+  assert_is_valid(o);
+
+  return to_oop(load_barrier(p, o));
 }
 
 template <DecoratorSet decorators, typename BarrierSetT>
 inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_in_heap_at(oop base, ptrdiff_t offset) {
-  oop* const addr = field_addr(base, offset);
-  const oop o = Raw::oop_load_in_heap(addr);
+  zpointer* const p = field_addr(base, offset);
+
+  const zpointer o = Raw::load_in_heap(p);
+  assert_is_valid(o);
 
   if (HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) {
-    return load_barrier_on_unknown_oop_field_preloaded(base, offset, addr, o);
+    return to_oop(load_barrier_on_unknown_oop_ref(base, offset, p, o));
   }
 
-  return load_barrier_on_oop_field_preloaded(addr, o);
+  return to_oop(load_barrier(p, o));
+}
+
+template <DecoratorSet decorators>
+bool is_store_barrier_no_keep_alive() {
+  if (HasDecorator<decorators, ON_STRONG_OOP_REF>::value) {
+    return HasDecorator<decorators, AS_NO_KEEPALIVE>::value;
+  }
+
+  if (HasDecorator<decorators, ON_WEAK_OOP_REF>::value) {
+    return true;
+  }
+
+  assert((decorators & ON_PHANTOM_OOP_REF) != 0, "Must be");
+  return true;
+}
+
+template <DecoratorSet decorators>
+inline bool is_store_barrier_no_keep_alive(oop base, ptrdiff_t offset) {
+  if (!HasDecorator<decorators, ON_UNKNOWN_OOP_REF>::value) {
+    return is_store_barrier_no_keep_alive<decorators>();
+  }
+
+  const DecoratorSet decorators_known_strength =
+      AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength<decorators>(base, offset);
+
+  if ((decorators_known_strength & ON_STRONG_OOP_REF) != 0) {
+    return (decorators & AS_NO_KEEPALIVE) != 0;
+  }
+
+  if ((decorators_known_strength & ON_WEAK_OOP_REF) != 0) {
+    return true;
+  }
+
+  assert((decorators_known_strength & ON_PHANTOM_OOP_REF) != 0, "Must be");
+  return true;
 }
 
 template <DecoratorSet decorators, typename BarrierSetT>
-template <typename T>
-inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(T* addr, oop compare_value, oop new_value) {
+inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap(zpointer* p, oop value) {
+  verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
+
+  if (is_store_barrier_no_keep_alive<decorators>()) {
+    no_keep_alive_store_barrier_heap(p);
+  } else {
+    store_barrier_heap_without_healing(p);
+  }
+
+  Raw::store_in_heap(p, store_good(value));
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_in_heap_at(oop base, ptrdiff_t offset, oop value) {
+  zpointer* const p = field_addr(base, offset);
+
+  if (is_store_barrier_no_keep_alive<decorators>(base, offset)) {
+    no_keep_alive_store_barrier_heap(p);
+  } else {
+    store_barrier_heap_without_healing(p);
+  }
+
+  Raw::store_in_heap(p, store_good(value));
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_store_not_in_heap(zpointer* p, oop value) {
+  verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
+
+  if (!is_store_barrier_no_keep_alive<decorators>()) {
+    store_barrier_native_without_healing(p);
+  }
+
+  Raw::store(p, store_good(value));
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_in_heap(zpointer* p, oop compare_value, oop new_value) {
   verify_decorators_present<ON_STRONG_OOP_REF>();
   verify_decorators_absent<AS_NO_KEEPALIVE>();
 
-  ZBarrier::load_barrier_on_oop_field(addr);
-  return Raw::oop_atomic_cmpxchg_in_heap(addr, compare_value, new_value);
+  store_barrier_heap_with_healing(p);
+
+  const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value));
+  assert_is_valid(o);
+
+  return to_oop(ZPointer::uncolor_store_good(o));
 }
 
 template <DecoratorSet decorators, typename BarrierSetT>
@@ -150,18 +283,27 @@ inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxc
   // calls with ON_UNKNOWN_OOP_REF set. However, we treat these as ON_STRONG_OOP_REF,
   // with the motivation that if you're doing Unsafe operations on a Reference.referent
   // field, then you're on your own anyway.
-  ZBarrier::load_barrier_on_oop_field(field_addr(base, offset));
-  return Raw::oop_atomic_cmpxchg_in_heap_at(base, offset, compare_value, new_value);
+  zpointer* const p = field_addr(base, offset);
+
+  store_barrier_heap_with_healing(p);
+
+  const zpointer o = Raw::atomic_cmpxchg_in_heap(p, store_good(compare_value), store_good(new_value));
+  assert_is_valid(o);
+
+  return to_oop(ZPointer::uncolor_store_good(o));
 }
 
 template <DecoratorSet decorators, typename BarrierSetT>
-template <typename T>
-inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(T* addr, oop new_value) {
+inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_in_heap(zpointer* p, oop new_value) {
   verify_decorators_present<ON_STRONG_OOP_REF>();
   verify_decorators_absent<AS_NO_KEEPALIVE>();
 
-  const oop o = Raw::oop_atomic_xchg_in_heap(addr, new_value);
-  return ZBarrier::load_barrier_on_oop(o);
+  store_barrier_heap_with_healing(p);
+
+  const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
+  assert_is_valid(o);
+
+  return to_oop(ZPointer::uncolor_store_good(o));
 }
 
 template <DecoratorSet decorators, typename BarrierSetT>
@@ -169,74 +311,189 @@ inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_
   verify_decorators_present<ON_STRONG_OOP_REF>();
   verify_decorators_absent<AS_NO_KEEPALIVE>();
 
-  const oop o = Raw::oop_atomic_xchg_in_heap_at(base, offset, new_value);
-  return ZBarrier::load_barrier_on_oop(o);
+  zpointer* const p = field_addr(base, offset);
+
+  store_barrier_heap_with_healing(p);
+
+  const zpointer o = Raw::atomic_xchg_in_heap(p, store_good(new_value));
+  assert_is_valid(o);
+
+  return to_oop(ZPointer::uncolor_store_good(o));
 }
 
 template <DecoratorSet decorators, typename BarrierSetT>
-template <typename T>
-inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw,
-                                                                                       arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw,
-                                                                                       size_t length) {
-  T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
-  T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
+inline zaddress ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_barriers(zpointer* dst, zpointer* src) {
+  store_barrier_heap_without_healing(dst);
 
-  if (!HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) {
-    // No check cast, bulk barrier and bulk copy
-    ZBarrier::load_barrier_on_oop_array(src, length);
-    return Raw::oop_arraycopy_in_heap(NULL, 0, src, NULL, 0, dst, length);
+  return ZBarrier::load_barrier_on_oop_field(src);
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one(zpointer* dst, zpointer* src) {
+  const zaddress obj = oop_copy_one_barriers(dst, src);
+
+  Atomic::store(dst, ZAddress::store_good(obj));
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_copy_one_check_cast(zpointer* dst, zpointer* src, Klass* dst_klass) {
+  const zaddress obj = oop_copy_one_barriers(dst, src);
+
+  if (!oopDesc::is_instanceof_or_null(to_oop(obj), dst_klass)) {
+    // Check cast failed
+    return false;
   }
 
+  Atomic::store(dst, ZAddress::store_good(obj));
+
+  return true;
+}
+
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_check_cast(zpointer* dst, zpointer* src, size_t length, Klass* dst_klass) {
   // Check cast and copy each elements
-  Klass* const dst_klass = objArrayOop(dst_obj)->element_klass();
-  for (const T* const end = src + length; src < end; src++, dst++) {
-    const oop elem = ZBarrier::load_barrier_on_oop_field(src);
-    if (!oopDesc::is_instanceof_or_null(elem, dst_klass)) {
+  for (const zpointer* const end = src + length; src < end; src++, dst++) {
+    if (!oop_copy_one_check_cast(dst, src, dst_klass)) {
       // Check cast failed
       return false;
     }
-
-    // Cast is safe, since we know it's never a narrowOop
-    *(oop*)dst = elem;
   }
 
   return true;
 }
 
+template <DecoratorSet decorators, typename BarrierSetT>
+inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap_no_check_cast(zpointer* dst, zpointer* src, size_t length) {
+  const bool is_disjoint = HasDecorator<decorators, ARRAYCOPY_DISJOINT>::value;
+
+  if (is_disjoint || src > dst) {
+    for (const zpointer* const end = src + length; src < end; src++, dst++) {
+      oop_copy_one(dst, src);
+    }
+    return true;
+  }
+
+  if (src < dst) {
+    const zpointer* const end = src;
+    src += length - 1;
+    dst += length - 1;
+    for ( ; src >= end; src--, dst--) {
+      oop_copy_one(dst, src);
+    }
+    return true;
+  }
+
+  // src and dst are the same; nothing to do
+  return true;
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline bool ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, zpointer* src_raw,
+                                                                                       arrayOop dst_obj, size_t dst_offset_in_bytes, zpointer* dst_raw,
+                                                                                       size_t length) {
+  zpointer* const src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw);
+  zpointer* const dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw);
+
+  if (HasDecorator<decorators, ARRAYCOPY_CHECKCAST>::value) {
+    Klass* const dst_klass = objArrayOop(dst_obj)->element_klass();
+    return oop_arraycopy_in_heap_check_cast(dst, src, length, dst_klass);
+  }
+
+  return oop_arraycopy_in_heap_no_check_cast(dst, src, length);
+}
+
+class ZStoreBarrierOopClosure : public BasicOopIterateClosure {
+public:
+  virtual void do_oop(oop* p_) {
+    volatile zpointer* const p = (volatile zpointer*)p_;
+    const zpointer ptr = ZBarrier::load_atomic(p);
+    const zaddress addr = ZPointer::uncolor(ptr);
+    ZBarrier::store_barrier_on_heap_oop_field(p, false /* heal */);
+    *p = ZAddress::store_good(addr);
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+};
+
+class ZLoadBarrierOopClosure : public BasicOopIterateClosure {
+public:
+  virtual void do_oop(oop* p) {
+    ZBarrier::load_barrier_on_oop_field((zpointer*)p);
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+};
+
 template <DecoratorSet decorators, typename BarrierSetT>
 inline void ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::clone_in_heap(oop src, oop dst, size_t size) {
-  ZBarrier::load_barrier_on_oop_fields(src);
+  assert_is_valid(to_zaddress(src));
+
+  // Fix the oops
+  ZLoadBarrierOopClosure cl;
+  ZIterator::oop_iterate(src, &cl);
+
+  // Clone the object
   Raw::clone_in_heap(src, dst, size);
+
+  assert(ZHeap::heap()->is_young(to_zaddress(dst)), "ZColorStoreGoodOopClosure is only valid for young objects");
+
+  // Color store good before handing out
+  ZStoreBarrierOopClosure cl_sg;
+  ZIterator::oop_iterate(dst, &cl_sg);
 }
 
 //
 // Not in heap
 //
 template <DecoratorSet decorators, typename BarrierSetT>
-template <typename T>
-inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(T* addr) {
+inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(zpointer* p) {
   verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
 
-  const oop o = Raw::oop_load_not_in_heap(addr);
-  return load_barrier_on_oop_field_preloaded(addr, o);
+  const zpointer o = Raw::template load<zpointer>(p);
+  assert_is_valid(o);
+  return to_oop(load_barrier(p, o));
 }
 
 template <DecoratorSet decorators, typename BarrierSetT>
-template <typename T>
-inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(T* addr, oop compare_value, oop new_value) {
-  verify_decorators_present<ON_STRONG_OOP_REF>();
-  verify_decorators_absent<AS_NO_KEEPALIVE>();
+inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_load_not_in_heap(oop* p) {
+  verify_decorators_absent<ON_UNKNOWN_OOP_REF>();
 
-  return Raw::oop_atomic_cmpxchg_not_in_heap(addr, compare_value, new_value);
+  if (HasDecorator<decorators, IN_NMETHOD>::value) {
+    return ZNMethod::load_oop(p, decorators);
+  } else {
+    return oop_load_not_in_heap((zpointer*)p);
+  }
 }
 
 template <DecoratorSet decorators, typename BarrierSetT>
-template <typename T>
-inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(T* addr, oop new_value) {
+inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_cmpxchg_not_in_heap(zpointer* p, oop compare_value, oop new_value) {
   verify_decorators_present<ON_STRONG_OOP_REF>();
   verify_decorators_absent<AS_NO_KEEPALIVE>();
 
-  return Raw::oop_atomic_xchg_not_in_heap(addr, new_value);
+  store_barrier_native_with_healing(p);
+
+  const zpointer o = Raw::atomic_cmpxchg(p, store_good(compare_value), store_good(new_value));
+  assert_is_valid(o);
+
+  return to_oop(ZPointer::uncolor_store_good(o));
+}
+
+template <DecoratorSet decorators, typename BarrierSetT>
+inline oop ZBarrierSet::AccessBarrier<decorators, BarrierSetT>::oop_atomic_xchg_not_in_heap(zpointer* p, oop new_value) {
+  verify_decorators_present<ON_STRONG_OOP_REF>();
+  verify_decorators_absent<AS_NO_KEEPALIVE>();
+
+  store_barrier_native_with_healing(p);
+
+  const zpointer o = Raw::atomic_xchg(p, store_good(new_value));
+  assert_is_valid(o);
+
+  return to_oop(ZPointer::uncolor_store_good(o));
 }
 
 #endif // SHARE_GC_Z_ZBARRIERSET_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zBarrierSetAssembler.cpp b/src/hotspot/share/gc/z/zBarrierSetAssembler.cpp
index 8127ff61079..e68142f0778 100644
--- a/src/hotspot/share/gc/z/zBarrierSetAssembler.cpp
+++ b/src/hotspot/share/gc/z/zBarrierSetAssembler.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,10 +26,18 @@
 #include "gc/z/zThreadLocalData.hpp"
 #include "runtime/javaThread.hpp"
 
-Address ZBarrierSetAssemblerBase::address_bad_mask_from_thread(Register thread) {
-  return Address(thread, ZThreadLocalData::address_bad_mask_offset());
+Address ZBarrierSetAssemblerBase::load_bad_mask_from_thread(Register thread) {
+  return Address(thread, ZThreadLocalData::load_bad_mask_offset());
 }
 
-Address ZBarrierSetAssemblerBase::address_bad_mask_from_jni_env(Register env) {
-  return Address(env, ZThreadLocalData::address_bad_mask_offset() - JavaThread::jni_environment_offset());
+Address ZBarrierSetAssemblerBase::mark_bad_mask_from_thread(Register thread) {
+  return Address(thread, ZThreadLocalData::mark_bad_mask_offset());
+}
+
+Address ZBarrierSetAssemblerBase::load_bad_mask_from_jni_env(Register env) {
+  return Address(env, ZThreadLocalData::load_bad_mask_offset() - JavaThread::jni_environment_offset());
+}
+
+Address ZBarrierSetAssemblerBase::mark_bad_mask_from_jni_env(Register env) {
+  return Address(env, ZThreadLocalData::mark_bad_mask_offset() - JavaThread::jni_environment_offset());
 }
diff --git a/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp b/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp
index e63c7a557d4..bcc757d6132 100644
--- a/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp
+++ b/src/hotspot/share/gc/z/zBarrierSetAssembler.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,8 +29,11 @@
 
 class ZBarrierSetAssemblerBase : public BarrierSetAssembler {
 public:
-  static Address address_bad_mask_from_thread(Register thread);
-  static Address address_bad_mask_from_jni_env(Register env);
+  static Address load_bad_mask_from_thread(Register thread);
+  static Address mark_bad_mask_from_thread(Register thread);
+
+  static Address load_bad_mask_from_jni_env(Register env);
+  static Address mark_bad_mask_from_jni_env(Register env);
 };
 
 // Needs to be included after definition of ZBarrierSetAssemblerBase
diff --git a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp
index 57c0a421f95..2c58b015564 100644
--- a/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp
+++ b/src/hotspot/share/gc/z/zBarrierSetNMethod.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,11 +23,16 @@
 
 #include "precompiled.hpp"
 #include "code/nmethod.hpp"
+#include "gc/shared/barrierSet.hpp"
+#include "gc/z/zAddress.hpp"
+#include "gc/z/zBarrier.inline.hpp"
+#include "gc/z/zBarrierSetAssembler.hpp"
 #include "gc/z/zBarrierSetNMethod.hpp"
-#include "gc/z/zGlobals.hpp"
 #include "gc/z/zLock.inline.hpp"
 #include "gc/z/zNMethod.hpp"
+#include "gc/z/zResurrection.inline.hpp"
 #include "gc/z/zThreadLocalData.hpp"
+#include "gc/z/zUncoloredRoot.inline.hpp"
 #include "logging/log.hpp"
 #include "runtime/threadWXSetters.inline.hpp"
 
@@ -35,7 +40,10 @@ bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) {
   ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
   log_trace(nmethod, barrier)("Entered critical zone for %p", nm);
 
+  log_develop_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by entry (try)", p2i(nm));
+
   if (!is_armed(nm)) {
+    log_develop_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by entry (disarmed)", p2i(nm));
     // Some other thread got here first and healed the oops
     // and disarmed the nmethod.
     return true;
@@ -44,6 +52,7 @@ bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) {
   MACOS_AARCH64_ONLY(ThreadWXEnable wx(WXWrite, Thread::current()));
 
   if (nm->is_unloading()) {
+    log_develop_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by entry (unloading)", p2i(nm));
     // We don't need to take the lock when unlinking nmethods from
     // the Method, because it is only concurrently unlinked by
     // the entry barrier, which acquires the per nmethod lock.
@@ -51,13 +60,20 @@ bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) {
 
     // We can end up calling nmethods that are unloading
     // since we clear compiled ICs lazily. Returning false
-    // will re-resovle the call and update the compiled IC.
+    // will re-resolve the call and update the compiled IC.
     return false;
   }
 
-  // Heal oops
-  ZNMethod::nmethod_oops_barrier(nm);
+  // Heal barriers
+  ZNMethod::nmethod_patch_barriers(nm);
 
+  // Heal oops
+  ZUncoloredRootProcessWeakOopClosure cl(ZNMethod::color(nm));
+  ZNMethod::nmethod_oops_do_inner(nm, &cl);
+
+  const uintptr_t prev_color = ZNMethod::color(nm);
+  const uintptr_t new_color = *(int*)ZPointerStoreGoodMaskLowOrderBitsAddr;
+  log_develop_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by entry (complete) [" PTR_FORMAT " -> " PTR_FORMAT "]", p2i(nm), prev_color, new_color);
 
   // CodeCache unloading support
   nm->mark_as_maybe_on_stack();
@@ -69,7 +85,7 @@ bool ZBarrierSetNMethod::nmethod_entry_barrier(nmethod* nm) {
 }
 
 int* ZBarrierSetNMethod::disarmed_guard_value_address() const {
-  return (int*)ZAddressBadMaskHighOrderBitsAddr;
+  return (int*)ZPointerStoreGoodMaskLowOrderBitsAddr;
 }
 
 ByteSize ZBarrierSetNMethod::thread_disarmed_guard_value_offset() const {
diff --git a/src/hotspot/share/gc/z/zBarrierSetRuntime.cpp b/src/hotspot/share/gc/z/zBarrierSetRuntime.cpp
index 895068c8cfd..da7adf7cc3a 100644
--- a/src/hotspot/share/gc/z/zBarrierSetRuntime.cpp
+++ b/src/hotspot/share/gc/z/zBarrierSetRuntime.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,31 +28,39 @@
 #include "runtime/interfaceSupport.inline.hpp"
 
 JRT_LEAF(oopDesc*, ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded(oopDesc* o, oop* p))
-  return ZBarrier::load_barrier_on_oop_field_preloaded(p, o);
+  return to_oop(ZBarrier::load_barrier_on_oop_field_preloaded((zpointer*)p, to_zpointer(o)));
 JRT_END
 
-JRT_LEAF(oopDesc*, ZBarrierSetRuntime::weak_load_barrier_on_oop_field_preloaded(oopDesc* o, oop* p))
-  return ZBarrier::weak_load_barrier_on_oop_field_preloaded(p, o);
-JRT_END
-
-JRT_LEAF(oopDesc*, ZBarrierSetRuntime::weak_load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p))
-  return ZBarrier::weak_load_barrier_on_weak_oop_field_preloaded(p, o);
-JRT_END
-
-JRT_LEAF(oopDesc*, ZBarrierSetRuntime::weak_load_barrier_on_phantom_oop_field_preloaded(oopDesc* o, oop* p))
-  return ZBarrier::weak_load_barrier_on_phantom_oop_field_preloaded(p, o);
+JRT_LEAF(zpointer, ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_store_good(oopDesc* o, oop* p))
+  return ZAddress::color(ZBarrier::load_barrier_on_oop_field_preloaded((zpointer*)p, to_zpointer(o)), ZPointerStoreGoodMask);
 JRT_END
 
 JRT_LEAF(oopDesc*, ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p))
-  return ZBarrier::load_barrier_on_weak_oop_field_preloaded(p, o);
+  return to_oop(ZBarrier::load_barrier_on_weak_oop_field_preloaded((zpointer*)p, to_zpointer(o)));
 JRT_END
 
 JRT_LEAF(oopDesc*, ZBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded(oopDesc* o, oop* p))
-  return ZBarrier::load_barrier_on_phantom_oop_field_preloaded(p, o);
+  return to_oop(ZBarrier::load_barrier_on_phantom_oop_field_preloaded((zpointer*)p, to_zpointer(o)));
 JRT_END
 
-JRT_LEAF(void, ZBarrierSetRuntime::load_barrier_on_oop_array(oop* p, size_t length))
-  ZBarrier::load_barrier_on_oop_array(p, length);
+JRT_LEAF(oopDesc*, ZBarrierSetRuntime::no_keepalive_load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p))
+  return to_oop(ZBarrier::no_keep_alive_load_barrier_on_weak_oop_field_preloaded((zpointer*)p, to_zpointer(o)));
+JRT_END
+
+JRT_LEAF(oopDesc*, ZBarrierSetRuntime::no_keepalive_load_barrier_on_phantom_oop_field_preloaded(oopDesc* o, oop* p))
+  return to_oop(ZBarrier::no_keep_alive_load_barrier_on_phantom_oop_field_preloaded((zpointer*)p, to_zpointer(o)));
+JRT_END
+
+JRT_LEAF(void, ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing(oop* p))
+  ZBarrier::store_barrier_on_heap_oop_field((zpointer*)p, true /* heal */);
+JRT_END
+
+JRT_LEAF(void, ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing(oop* p))
+  ZBarrier::store_barrier_on_heap_oop_field((zpointer*)p, false /* heal */);
+JRT_END
+
+JRT_LEAF(void, ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing(oop* p))
+  ZBarrier::store_barrier_on_native_oop_field((zpointer*)p, false /* heal */);
 JRT_END
 
 JRT_LEAF(void, ZBarrierSetRuntime::clone(oopDesc* src, oopDesc* dst, size_t size))
@@ -60,22 +68,23 @@ JRT_LEAF(void, ZBarrierSetRuntime::clone(oopDesc* src, oopDesc* dst, size_t size
 JRT_END
 
 address ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr(DecoratorSet decorators) {
-  if (decorators & ON_PHANTOM_OOP_REF) {
-    if (decorators & AS_NO_KEEPALIVE) {
-      return weak_load_barrier_on_phantom_oop_field_preloaded_addr();
+  if (decorators & AS_NO_KEEPALIVE) {
+    if (decorators & ON_PHANTOM_OOP_REF) {
+      return no_keepalive_load_barrier_on_phantom_oop_field_preloaded_addr();
+    } else if (decorators & ON_WEAK_OOP_REF) {
+      return no_keepalive_load_barrier_on_weak_oop_field_preloaded_addr();
     } else {
-      return load_barrier_on_phantom_oop_field_preloaded_addr();
-    }
-  } else if (decorators & ON_WEAK_OOP_REF) {
-    if (decorators & AS_NO_KEEPALIVE) {
-      return weak_load_barrier_on_weak_oop_field_preloaded_addr();
-    } else {
-      return load_barrier_on_weak_oop_field_preloaded_addr();
+      assert((decorators & ON_STRONG_OOP_REF), "Expected type");
+      // Normal loads on strong oop never keep objects alive
+      return load_barrier_on_oop_field_preloaded_addr();
     }
   } else {
-    if (decorators & AS_NO_KEEPALIVE) {
-      return weak_load_barrier_on_oop_field_preloaded_addr();
+    if (decorators & ON_PHANTOM_OOP_REF) {
+      return load_barrier_on_phantom_oop_field_preloaded_addr();
+    } else if (decorators & ON_WEAK_OOP_REF) {
+      return load_barrier_on_weak_oop_field_preloaded_addr();
     } else {
+      assert((decorators & ON_STRONG_OOP_REF), "Expected type");
       return load_barrier_on_oop_field_preloaded_addr();
     }
   }
@@ -85,6 +94,10 @@ address ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr() {
   return reinterpret_cast<address>(load_barrier_on_oop_field_preloaded);
 }
 
+address ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_store_good_addr() {
+  return reinterpret_cast<address>(load_barrier_on_oop_field_preloaded_store_good);
+}
+
 address ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr() {
   return reinterpret_cast<address>(load_barrier_on_weak_oop_field_preloaded);
 }
@@ -93,20 +106,24 @@ address ZBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded_addr() {
   return reinterpret_cast<address>(load_barrier_on_phantom_oop_field_preloaded);
 }
 
-address ZBarrierSetRuntime::weak_load_barrier_on_oop_field_preloaded_addr() {
-  return reinterpret_cast<address>(weak_load_barrier_on_oop_field_preloaded);
+address ZBarrierSetRuntime::no_keepalive_load_barrier_on_weak_oop_field_preloaded_addr() {
+  return reinterpret_cast<address>(no_keepalive_load_barrier_on_weak_oop_field_preloaded);
 }
 
-address ZBarrierSetRuntime::weak_load_barrier_on_weak_oop_field_preloaded_addr() {
-  return reinterpret_cast<address>(weak_load_barrier_on_weak_oop_field_preloaded);
+address ZBarrierSetRuntime::no_keepalive_load_barrier_on_phantom_oop_field_preloaded_addr() {
+  return reinterpret_cast<address>(no_keepalive_load_barrier_on_phantom_oop_field_preloaded);
 }
 
-address ZBarrierSetRuntime::weak_load_barrier_on_phantom_oop_field_preloaded_addr() {
-  return reinterpret_cast<address>(weak_load_barrier_on_phantom_oop_field_preloaded);
+address ZBarrierSetRuntime::store_barrier_on_oop_field_with_healing_addr() {
+  return reinterpret_cast<address>(store_barrier_on_oop_field_with_healing);
 }
 
-address ZBarrierSetRuntime::load_barrier_on_oop_array_addr() {
-  return reinterpret_cast<address>(load_barrier_on_oop_array);
+address ZBarrierSetRuntime::store_barrier_on_oop_field_without_healing_addr() {
+  return reinterpret_cast<address>(store_barrier_on_oop_field_without_healing);
+}
+
+address ZBarrierSetRuntime::store_barrier_on_native_oop_field_without_healing_addr() {
+  return reinterpret_cast<address>(store_barrier_on_native_oop_field_without_healing);
 }
 
 address ZBarrierSetRuntime::clone_addr() {
diff --git a/src/hotspot/share/gc/z/zBarrierSetRuntime.hpp b/src/hotspot/share/gc/z/zBarrierSetRuntime.hpp
index b3a143de3e9..a569ff3c158 100644
--- a/src/hotspot/share/gc/z/zBarrierSetRuntime.hpp
+++ b/src/hotspot/share/gc/z/zBarrierSetRuntime.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,32 +24,36 @@
 #ifndef SHARE_GC_Z_ZBARRIERSETRUNTIME_HPP
 #define SHARE_GC_Z_ZBARRIERSETRUNTIME_HPP
 
+#include "gc/z/zAddress.hpp"
 #include "memory/allStatic.hpp"
 #include "oops/accessDecorators.hpp"
+#include "oops/oopsHierarchy.hpp"
 #include "utilities/globalDefinitions.hpp"
 
-class oopDesc;
-
 class ZBarrierSetRuntime : public AllStatic {
 private:
   static oopDesc* load_barrier_on_oop_field_preloaded(oopDesc* o, oop* p);
+  static zpointer load_barrier_on_oop_field_preloaded_store_good(oopDesc* o, oop* p);
   static oopDesc* load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p);
   static oopDesc* load_barrier_on_phantom_oop_field_preloaded(oopDesc* o, oop* p);
-  static oopDesc* weak_load_barrier_on_oop_field_preloaded(oopDesc* o, oop* p);
-  static oopDesc* weak_load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p);
-  static oopDesc* weak_load_barrier_on_phantom_oop_field_preloaded(oopDesc* o, oop* p);
-  static void load_barrier_on_oop_array(oop* p, size_t length);
+  static oopDesc* no_keepalive_load_barrier_on_weak_oop_field_preloaded(oopDesc* o, oop* p);
+  static oopDesc* no_keepalive_load_barrier_on_phantom_oop_field_preloaded(oopDesc* o, oop* p);
+  static void store_barrier_on_oop_field_with_healing(oop* p);
+  static void store_barrier_on_oop_field_without_healing(oop* p);
+  static void store_barrier_on_native_oop_field_without_healing(oop* p);
   static void clone(oopDesc* src, oopDesc* dst, size_t size);
 
 public:
   static address load_barrier_on_oop_field_preloaded_addr(DecoratorSet decorators);
   static address load_barrier_on_oop_field_preloaded_addr();
+  static address load_barrier_on_oop_field_preloaded_store_good_addr();
   static address load_barrier_on_weak_oop_field_preloaded_addr();
   static address load_barrier_on_phantom_oop_field_preloaded_addr();
-  static address weak_load_barrier_on_oop_field_preloaded_addr();
-  static address weak_load_barrier_on_weak_oop_field_preloaded_addr();
-  static address weak_load_barrier_on_phantom_oop_field_preloaded_addr();
-  static address load_barrier_on_oop_array_addr();
+  static address no_keepalive_load_barrier_on_weak_oop_field_preloaded_addr();
+  static address no_keepalive_load_barrier_on_phantom_oop_field_preloaded_addr();
+  static address store_barrier_on_oop_field_with_healing_addr();
+  static address store_barrier_on_oop_field_without_healing_addr();
+  static address store_barrier_on_native_oop_field_without_healing_addr();
   static address clone_addr();
 };
 
diff --git a/src/hotspot/share/gc/z/zBarrierSetStackChunk.cpp b/src/hotspot/share/gc/z/zBarrierSetStackChunk.cpp
index 0b1452746f2..e33b86ef22b 100644
--- a/src/hotspot/share/gc/z/zBarrierSetStackChunk.cpp
+++ b/src/hotspot/share/gc/z/zBarrierSetStackChunk.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,25 +23,25 @@
  */
 
 #include "precompiled.hpp"
-#include "gc/z/zBarrier.inline.hpp"
 #include "gc/z/zBarrierSetStackChunk.hpp"
-#include "runtime/atomic.hpp"
+#include "gc/z/zContinuation.hpp"
 #include "utilities/debug.hpp"
 
 void ZBarrierSetStackChunk::encode_gc_mode(stackChunkOop chunk, OopIterator* iterator) {
-  // Do nothing
+  ZContinuation::ZColorStackOopClosure cl(chunk);
+  iterator->oops_do(&cl);
 }
 
 void ZBarrierSetStackChunk::decode_gc_mode(stackChunkOop chunk, OopIterator* iterator) {
-  // Do nothing
+  ZContinuation::ZUncolorStackOopClosure cl;
+  iterator->oops_do(&cl);
 }
 
 oop ZBarrierSetStackChunk::load_oop(stackChunkOop chunk, oop* addr) {
-  oop obj = Atomic::load(addr);
-  return ZBarrier::load_barrier_on_oop_field_preloaded((volatile oop*)NULL, obj);
+  return ZContinuation::load_oop(chunk, addr);
 }
 
 oop ZBarrierSetStackChunk::load_oop(stackChunkOop chunk, narrowOop* addr) {
   ShouldNotReachHere();
-  return NULL;
+  return nullptr;
 }
diff --git a/src/hotspot/share/gc/z/zBitMap.hpp b/src/hotspot/share/gc/z/zBitMap.hpp
index dc00fc19981..3568723383c 100644
--- a/src/hotspot/share/gc/z/zBitMap.hpp
+++ b/src/hotspot/share/gc/z/zBitMap.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,12 @@
 
 #include "utilities/bitMap.hpp"
 
+class ZMovableBitMap : public CHeapBitMap {
+public:
+  ZMovableBitMap();
+  ZMovableBitMap(ZMovableBitMap&& bitmap);
+};
+
 class ZBitMap : public CHeapBitMap {
 private:
   static bm_word_t bit_mask_pair(idx_t bit);
@@ -35,8 +41,26 @@ private:
 
 public:
   ZBitMap(idx_t size_in_bits);
+  ZBitMap(const ZBitMap& other);
 
   bool par_set_bit_pair(idx_t bit, bool finalizable, bool& inc_live);
+
+  class ReverseIterator;
+};
+
+class ZBitMap::ReverseIterator {
+  BitMap* const _bitmap;
+  BitMap::idx_t _beg;
+  BitMap::idx_t _end;
+
+public:
+  ReverseIterator(BitMap* bitmap);
+  ReverseIterator(BitMap* bitmap, BitMap::idx_t beg, BitMap::idx_t end);
+
+  void reset(BitMap::idx_t beg, BitMap::idx_t end);
+  void reset(BitMap::idx_t end);
+
+  bool next(BitMap::idx_t* index);
 };
 
 #endif // SHARE_GC_Z_ZBITMAP_HPP
diff --git a/src/hotspot/share/gc/z/zBitMap.inline.hpp b/src/hotspot/share/gc/z/zBitMap.inline.hpp
index f757cec2706..79d78604087 100644
--- a/src/hotspot/share/gc/z/zBitMap.inline.hpp
+++ b/src/hotspot/share/gc/z/zBitMap.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,9 +30,23 @@
 #include "utilities/bitMap.inline.hpp"
 #include "utilities/debug.hpp"
 
+inline ZMovableBitMap::ZMovableBitMap() :
+    CHeapBitMap(mtGC) {}
+
+inline ZMovableBitMap::ZMovableBitMap(ZMovableBitMap&& bitmap) :
+    CHeapBitMap(mtGC) {
+  update(bitmap.map(), bitmap.size());
+  bitmap.update(nullptr, 0);
+}
+
 inline ZBitMap::ZBitMap(idx_t size_in_bits) :
     CHeapBitMap(size_in_bits, mtGC, false /* clear */) {}
 
+inline ZBitMap::ZBitMap(const ZBitMap& other) :
+    CHeapBitMap(other.size(), mtGC, false /* clear */) {
+  memcpy(map(), other.map(), size_in_bytes());
+}
+
 inline BitMap::bm_word_t ZBitMap::bit_mask_pair(idx_t bit) {
   assert(bit_in_word(bit) < BitsPerWord - 1, "Invalid bit index");
   return (bm_word_t)3 << bit_in_word(bit);
@@ -77,4 +91,34 @@ inline bool ZBitMap::par_set_bit_pair(idx_t bit, bool finalizable, bool& inc_liv
   }
 }
 
+inline ZBitMap::ReverseIterator::ReverseIterator(BitMap* bitmap) :
+    ZBitMap::ReverseIterator(bitmap, 0, bitmap->size()) {}
+
+inline ZBitMap::ReverseIterator::ReverseIterator(BitMap* bitmap, BitMap::idx_t beg, BitMap::idx_t end) :
+    _bitmap(bitmap),
+    _beg(beg),
+    _end(end) {}
+
+inline void ZBitMap::ReverseIterator::reset(BitMap::idx_t beg, BitMap::idx_t end) {
+  assert(beg < _bitmap->size(), "beg index out of bounds");
+  assert(end >= beg && end <= _bitmap->size(), "end index out of bounds");
+  _beg = beg;
+  _end = end;
+}
+
+inline void ZBitMap::ReverseIterator::reset(BitMap::idx_t end) {
+  assert(end >= _beg && end <= _bitmap->size(), "end index out of bounds");
+  _end = end;
+}
+
+inline bool ZBitMap::ReverseIterator::next(BitMap::idx_t *index) {
+  BitMap::ReverseIterator iter(*_bitmap, _beg, _end);
+  if (iter.is_empty()) {
+    return false;
+  }
+
+  *index = _end = iter.index();
+  return true;
+}
+
 #endif // SHARE_GC_Z_ZBITMAP_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zCPU.cpp b/src/hotspot/share/gc/z/zCPU.cpp
index ba9da1d6e33..f6aaa96476f 100644
--- a/src/hotspot/share/gc/z/zCPU.cpp
+++ b/src/hotspot/share/gc/z/zCPU.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,12 +32,12 @@
 #define ZCPU_UNKNOWN_AFFINITY ((Thread*)-1)
 #define ZCPU_UNKNOWN_SELF     ((Thread*)-2)
 
-PaddedEnd<ZCPU::ZCPUAffinity>* ZCPU::_affinity = NULL;
+PaddedEnd<ZCPU::ZCPUAffinity>* ZCPU::_affinity = nullptr;
 THREAD_LOCAL Thread*           ZCPU::_self     = ZCPU_UNKNOWN_SELF;
 THREAD_LOCAL uint32_t          ZCPU::_cpu      = 0;
 
 void ZCPU::initialize() {
-  assert(_affinity == NULL, "Already initialized");
+  assert(_affinity == nullptr, "Already initialized");
   const uint32_t ncpus = count();
 
   _affinity = PaddedArray<ZCPUAffinity, mtGC>::create_unfreeable(ncpus);
diff --git a/src/hotspot/share/gc/z/zCPU.inline.hpp b/src/hotspot/share/gc/z/zCPU.inline.hpp
index 45c7e36568e..67d26f4c2e1 100644
--- a/src/hotspot/share/gc/z/zCPU.inline.hpp
+++ b/src/hotspot/share/gc/z/zCPU.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,7 +34,7 @@ inline uint32_t ZCPU::count() {
 }
 
 inline uint32_t ZCPU::id() {
-  assert(_affinity != NULL, "Not initialized");
+  assert(_affinity != nullptr, "Not initialized");
 
   // Fast path
   if (_affinity[_cpu]._thread == _self) {
diff --git a/src/hotspot/share/gc/z/zCollectedHeap.cpp b/src/hotspot/share/gc/z/zCollectedHeap.cpp
index c4547cb2fa4..749179b26cf 100644
--- a/src/hotspot/share/gc/z/zCollectedHeap.cpp
+++ b/src/hotspot/share/gc/z/zCollectedHeap.cpp
@@ -21,20 +21,27 @@
  * questions.
  */
 
+#include "gc/z/zAddress.hpp"
 #include "precompiled.hpp"
 #include "classfile/classLoaderData.hpp"
 #include "gc/shared/gcHeapSummary.hpp"
-#include "gc/shared/gcLocker.inline.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
+#include "gc/z/zAbort.hpp"
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zAllocator.inline.hpp"
 #include "gc/z/zCollectedHeap.hpp"
+#include "gc/z/zContinuation.inline.hpp"
 #include "gc/z/zDirector.hpp"
 #include "gc/z/zDriver.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zHeap.inline.hpp"
+#include "gc/z/zJNICritical.hpp"
 #include "gc/z/zNMethod.hpp"
 #include "gc/z/zObjArrayAllocator.hpp"
-#include "gc/z/zOop.inline.hpp"
 #include "gc/z/zServiceability.hpp"
+#include "gc/z/zStackChunkGCData.inline.hpp"
 #include "gc/z/zStat.hpp"
 #include "gc/z/zUtils.inline.hpp"
 #include "memory/classLoaderMetaspace.hpp"
@@ -43,6 +50,8 @@
 #include "memory/universe.hpp"
 #include "oops/stackChunkOop.hpp"
 #include "runtime/continuationJavaClasses.hpp"
+#include "runtime/jniHandles.inline.hpp"
+#include "services/memoryUsage.hpp"
 #include "utilities/align.hpp"
 
 ZCollectedHeap* ZCollectedHeap::heap() {
@@ -54,8 +63,9 @@ ZCollectedHeap::ZCollectedHeap() :
     _barrier_set(),
     _initialize(&_barrier_set),
     _heap(),
-    _driver(new ZDriver()),
-    _director(new ZDirector(_driver)),
+    _driver_minor(new ZDriverMinor()),
+    _driver_major(new ZDriverMajor()),
+    _director(new ZDirector()),
     _stat(new ZStat()),
     _runtime_workers() {}
 
@@ -72,7 +82,7 @@ jint ZCollectedHeap::initialize() {
     return JNI_ENOMEM;
   }
 
-  Universe::calculate_verify_data((HeapWord*)0, (HeapWord*)UINTPTR_MAX);
+  Universe::set_verify_data(~(ZAddressHeapBase - 1) | 0x7, ZAddressHeapBase);
 
   return JNI_OK;
 }
@@ -91,6 +101,8 @@ public:
 };
 
 void ZCollectedHeap::stop() {
+  log_info_p(gc, exit)("Stopping ZGC");
+  ZAbort::abort();
   ZStopConcurrentGCThreadClosure cl;
   gc_threads_do(&cl);
 }
@@ -126,45 +138,28 @@ bool ZCollectedHeap::is_in(const void* p) const {
 }
 
 bool ZCollectedHeap::requires_barriers(stackChunkOop obj) const {
-  uintptr_t* cont_addr = obj->field_addr<uintptr_t>(jdk_internal_vm_StackChunk::cont_offset());
-
-  if (!_heap.is_allocating(cast_from_oop<uintptr_t>(obj))) {
-    // An object that isn't allocating, is visible from GC tracing. Such
-    // stack chunks require barriers.
-    return true;
-  }
-
-  if (!ZAddress::is_good_or_null(*cont_addr)) {
-    // If a chunk is allocated after a GC started, but before relocate start
-    // we can have an allocating chunk that isn't deeply good. That means that
-    // the contained oops might be bad and require GC barriers.
-    return true;
-  }
-
-  // The chunk is allocating and its pointers are good. This chunk needs no
-  // GC barriers
-  return false;
+  return ZContinuation::requires_barriers(&_heap, obj);
 }
 
 HeapWord* ZCollectedHeap::allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) {
   const size_t size_in_bytes = ZUtils::words_to_bytes(align_object_size(requested_size));
-  const uintptr_t addr = _heap.alloc_tlab(size_in_bytes);
+  const zaddress addr = ZAllocator::eden()->alloc_tlab(size_in_bytes);
 
-  if (addr != 0) {
+  if (!is_null(addr)) {
     *actual_size = requested_size;
   }
 
-  return (HeapWord*)addr;
+  return (HeapWord*)untype(addr);
 }
 
 oop ZCollectedHeap::array_allocate(Klass* klass, size_t size, int length, bool do_zero, TRAPS) {
-  ZObjArrayAllocator allocator(klass, size, length, do_zero, THREAD);
+  const ZObjArrayAllocator allocator(klass, size, length, do_zero, THREAD);
   return allocator.allocate();
 }
 
 HeapWord* ZCollectedHeap::mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) {
   const size_t size_in_bytes = ZUtils::words_to_bytes(align_object_size(size));
-  return (HeapWord*)_heap.alloc_object(size_in_bytes);
+  return (HeapWord*)ZAllocator::eden()->alloc_object(size_in_bytes);
 }
 
 MetaWord* ZCollectedHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data,
@@ -175,7 +170,7 @@ MetaWord* ZCollectedHeap::satisfy_failed_metadata_allocation(ClassLoaderData* lo
 
   // Expand and retry allocation
   MetaWord* const result = loader_data->metaspace_non_null()->expand_and_allocate(size, mdtype);
-  if (result != NULL) {
+  if (result != nullptr) {
     return result;
   }
 
@@ -184,17 +179,47 @@ MetaWord* ZCollectedHeap::satisfy_failed_metadata_allocation(ClassLoaderData* lo
 }
 
 void ZCollectedHeap::collect(GCCause::Cause cause) {
-  _driver->collect(cause);
+  // Handle external collection requests
+  switch (cause) {
+  case GCCause::_wb_young_gc:
+  case GCCause::_scavenge_alot:
+    // Start urgent minor GC
+    _driver_minor->collect(ZDriverRequest(cause, ZYoungGCThreads, 0));
+    break;
+
+  case GCCause::_heap_dump:
+  case GCCause::_heap_inspection:
+  case GCCause::_wb_full_gc:
+  case GCCause::_wb_breakpoint:
+  case GCCause::_dcmd_gc_run:
+  case GCCause::_java_lang_system_gc:
+  case GCCause::_full_gc_alot:
+  case GCCause::_jvmti_force_gc:
+  case GCCause::_metadata_GC_clear_soft_refs:
+  case GCCause::_codecache_GC_aggressive:
+    // Start urgent major GC
+    _driver_major->collect(ZDriverRequest(cause, ZYoungGCThreads, ZOldGCThreads));
+    break;
+
+  case GCCause::_metadata_GC_threshold:
+  case GCCause::_codecache_GC_threshold:
+    // Start not urgent major GC
+    _driver_major->collect(ZDriverRequest(cause, 1, 1));
+    break;
+
+  default:
+    fatal("Unsupported GC cause (%s)", GCCause::to_string(cause));
+    break;
+  }
 }
 
 void ZCollectedHeap::collect_as_vm_thread(GCCause::Cause cause) {
   // These collection requests are ignored since ZGC can't run a synchronous
   // GC cycle from within the VM thread. This is considered benign, since the
   // only GC causes coming in here should be heap dumper and heap inspector.
-  // However, neither the heap dumper nor the heap inspector really need a GC
-  // to happen, but the result of their heap iterations might in that case be
-  // less accurate since they might include objects that would otherwise have
-  // been collected by a GC.
+  // If the heap dumper or heap inspector explicitly requests a gc and the
+  // caller is not the VM thread a synchronous GC cycle is performed from the
+  // caller thread in the prologue.
   assert(Thread::current()->is_VM_thread(), "Should be the VM thread");
   guarantee(cause == GCCause::_heap_dump ||
             cause == GCCause::_heap_inspection, "Invalid cause");
@@ -226,19 +251,27 @@ bool ZCollectedHeap::uses_stack_watermark_barrier() const {
 }
 
 MemoryUsage ZCollectedHeap::memory_usage() {
-  return _heap.serviceability_memory_pool()->get_memory_usage();
+  const size_t initial_size = ZHeap::heap()->initial_capacity();
+  const size_t committed    = ZHeap::heap()->capacity();
+  const size_t used         = MIN2(ZHeap::heap()->used(), committed);
+  const size_t max_size     = ZHeap::heap()->max_capacity();
+
+  return MemoryUsage(initial_size, used, committed, max_size);
 }
 
 GrowableArray<GCMemoryManager*> ZCollectedHeap::memory_managers() {
-  GrowableArray<GCMemoryManager*> memory_managers(2);
-  memory_managers.append(_heap.serviceability_cycle_memory_manager());
-  memory_managers.append(_heap.serviceability_pause_memory_manager());
+  GrowableArray<GCMemoryManager*> memory_managers(4);
+  memory_managers.append(_heap.serviceability_cycle_memory_manager(true /* minor */));
+  memory_managers.append(_heap.serviceability_cycle_memory_manager(false /* minor */));
+  memory_managers.append(_heap.serviceability_pause_memory_manager(true /* minor */));
+  memory_managers.append(_heap.serviceability_pause_memory_manager(false /* minor */));
   return memory_managers;
 }
 
 GrowableArray<MemoryPool*> ZCollectedHeap::memory_pools() {
-  GrowableArray<MemoryPool*> memory_pools(1);
-  memory_pools.append(_heap.serviceability_memory_pool());
+  GrowableArray<MemoryPool*> memory_pools(2);
+  memory_pools.append(_heap.serviceability_memory_pool(ZGenerationId::young));
+  memory_pools.append(_heap.serviceability_memory_pool(ZGenerationId::old));
   return memory_pools;
 }
 
@@ -250,6 +283,14 @@ ParallelObjectIteratorImpl* ZCollectedHeap::parallel_object_iterator(uint nworke
   return _heap.parallel_object_iterator(nworkers, true /* visit_weaks */);
 }
 
+void ZCollectedHeap::pin_object(JavaThread* thread, oop obj) {
+  ZJNICritical::enter(thread);
+}
+
+void ZCollectedHeap::unpin_object(JavaThread* thread, oop obj) {
+  ZJNICritical::exit(thread);
+}
+
 void ZCollectedHeap::keep_alive(oop obj) {
   _heap.keep_alive(obj);
 }
@@ -259,7 +300,14 @@ void ZCollectedHeap::register_nmethod(nmethod* nm) {
 }
 
 void ZCollectedHeap::unregister_nmethod(nmethod* nm) {
-  ZNMethod::unregister_nmethod(nm);
+  // ZGC follows the 'unlink | handshake | purge', where nmethods are unlinked
+  // from the system, threads are handshaked so that no reference to the
+  // unlinked nmethods exist, then the nmethods are deleted in the purge phase.
+  //
+  // CollectedHeap::unregister_nmethod is called during the flush phase, which
+  // is too late for ZGC.
+
+  ZNMethod::purge_nmethod(nm);
 }
 
 void ZCollectedHeap::verify_nmethod(nmethod* nm) {
@@ -272,30 +320,39 @@ WorkerThreads* ZCollectedHeap::safepoint_workers() {
 
 void ZCollectedHeap::gc_threads_do(ThreadClosure* tc) const {
   tc->do_thread(_director);
-  tc->do_thread(_driver);
+  tc->do_thread(_driver_major);
+  tc->do_thread(_driver_minor);
   tc->do_thread(_stat);
   _heap.threads_do(tc);
   _runtime_workers.threads_do(tc);
 }
 
 VirtualSpaceSummary ZCollectedHeap::create_heap_space_summary() {
-  return VirtualSpaceSummary((HeapWord*)0, (HeapWord*)capacity(), (HeapWord*)max_capacity());
+  const uintptr_t start = ZAddressHeapBase;
+
+  // Fake values. ZGC does not commit memory contiguously in the reserved
+  // address space, and the reserved space is larger than MaxHeapSize.
+  const uintptr_t committed_end = ZAddressHeapBase + capacity();
+  const uintptr_t reserved_end = ZAddressHeapBase + max_capacity();
+
+  return VirtualSpaceSummary((HeapWord*)start, (HeapWord*)committed_end, (HeapWord*)reserved_end);
+}
+
+bool ZCollectedHeap::contains_null(const oop* p) const {
+  const zpointer* const ptr = (const zpointer*)p;
+  return is_null_any(*ptr);
 }
 
 void ZCollectedHeap::safepoint_synchronize_begin() {
+  ZGeneration::young()->synchronize_relocation();
+  ZGeneration::old()->synchronize_relocation();
   SuspendibleThreadSet::synchronize();
 }
 
 void ZCollectedHeap::safepoint_synchronize_end() {
   SuspendibleThreadSet::desynchronize();
-}
-
-void ZCollectedHeap::pin_object(JavaThread* thread, oop obj) {
-  GCLocker::lock_critical(thread);
-}
-
-void ZCollectedHeap::unpin_object(JavaThread* thread, oop obj) {
-  GCLocker::unlock_critical(thread);
+  ZGeneration::old()->desynchronize_relocation();
+  ZGeneration::young()->desynchronize_relocation();
 }
 
 void ZCollectedHeap::prepare_for_verify() {
@@ -308,21 +365,29 @@ void ZCollectedHeap::print_on(outputStream* st) const {
 
 void ZCollectedHeap::print_on_error(outputStream* st) const {
   st->print_cr("ZGC Globals:");
-  st->print_cr(" GlobalPhase:       %u (%s)", ZGlobalPhase, ZGlobalPhaseToString());
-  st->print_cr(" GlobalSeqNum:      %u", ZGlobalSeqNum);
-  st->print_cr(" Offset Max:        " SIZE_FORMAT "%s (" PTR_FORMAT ")",
+  st->print_cr(" Young Collection:   %s/%u", ZGeneration::young()->phase_to_string(), ZGeneration::young()->seqnum());
+  st->print_cr(" Old Collection:     %s/%u", ZGeneration::old()->phase_to_string(), ZGeneration::old()->seqnum());
+  st->print_cr(" Offset Max:         " SIZE_FORMAT "%s (" PTR_FORMAT ")",
                byte_size_in_exact_unit(ZAddressOffsetMax),
                exact_unit_for_byte_size(ZAddressOffsetMax),
                ZAddressOffsetMax);
-  st->print_cr(" Page Size Small:   " SIZE_FORMAT "M", ZPageSizeSmall / M);
-  st->print_cr(" Page Size Medium:  " SIZE_FORMAT "M", ZPageSizeMedium / M);
+  st->print_cr(" Page Size Small:    " SIZE_FORMAT "M", ZPageSizeSmall / M);
+  st->print_cr(" Page Size Medium:   " SIZE_FORMAT "M", ZPageSizeMedium / M);
   st->cr();
   st->print_cr("ZGC Metadata Bits:");
-  st->print_cr(" Good:              " PTR_FORMAT, ZAddressGoodMask);
-  st->print_cr(" Bad:               " PTR_FORMAT, ZAddressBadMask);
-  st->print_cr(" WeakBad:           " PTR_FORMAT, ZAddressWeakBadMask);
-  st->print_cr(" Marked:            " PTR_FORMAT, ZAddressMetadataMarked);
-  st->print_cr(" Remapped:          " PTR_FORMAT, ZAddressMetadataRemapped);
+  st->print_cr(" LoadGood:           " PTR_FORMAT, ZPointerLoadGoodMask);
+  st->print_cr(" LoadBad:            " PTR_FORMAT, ZPointerLoadBadMask);
+  st->print_cr(" MarkGood:           " PTR_FORMAT, ZPointerMarkGoodMask);
+  st->print_cr(" MarkBad:            " PTR_FORMAT, ZPointerMarkBadMask);
+  st->print_cr(" StoreGood:          " PTR_FORMAT, ZPointerStoreGoodMask);
+  st->print_cr(" StoreBad:           " PTR_FORMAT, ZPointerStoreBadMask);
+  st->print_cr(" ------------------- ");
+  st->print_cr(" Remapped:           " PTR_FORMAT, ZPointerRemapped);
+  st->print_cr(" RemappedYoung:      " PTR_FORMAT, ZPointerRemappedYoungMask);
+  st->print_cr(" RemappedOld:        " PTR_FORMAT, ZPointerRemappedOldMask);
+  st->print_cr(" MarkedYoung:        " PTR_FORMAT, ZPointerMarkedYoung);
+  st->print_cr(" MarkedOld:          " PTR_FORMAT, ZPointerMarkedOld);
+  st->print_cr(" Remembered:         " PTR_FORMAT, ZPointerRemembered);
   st->cr();
   CollectedHeap::print_on_error(st);
 }
@@ -340,11 +405,11 @@ bool ZCollectedHeap::print_location(outputStream* st, void* addr) const {
 }
 
 void ZCollectedHeap::verify(VerifyOption option /* ignored */) {
-  _heap.verify();
+  fatal("Externally triggered verification not supported");
 }
 
 bool ZCollectedHeap::is_oop(oop object) const {
-  return _heap.is_oop(ZOop::to_address(object));
+  return _heap.is_oop(cast_from_oop<uintptr_t>(object));
 }
 
 bool ZCollectedHeap::supports_concurrent_gc_breakpoints() const {
diff --git a/src/hotspot/share/gc/z/zCollectedHeap.hpp b/src/hotspot/share/gc/z/zCollectedHeap.hpp
index d89d6917667..2b83f614009 100644
--- a/src/hotspot/share/gc/z/zCollectedHeap.hpp
+++ b/src/hotspot/share/gc/z/zCollectedHeap.hpp
@@ -34,7 +34,8 @@
 #include "services/memoryUsage.hpp"
 
 class ZDirector;
-class ZDriver;
+class ZDriverMajor;
+class ZDriverMinor;
 class ZStat;
 
 class ZCollectedHeap : public CollectedHeap {
@@ -45,7 +46,8 @@ private:
   ZBarrierSet       _barrier_set;
   ZInitialize       _initialize;
   ZHeap             _heap;
-  ZDriver*          _driver;
+  ZDriverMinor*     _driver_minor;
+  ZDriverMajor*     _driver_major;
   ZDirector*        _director;
   ZStat*            _stat;
   ZRuntimeWorkers   _runtime_workers;
@@ -110,6 +112,8 @@ public:
 
   VirtualSpaceSummary create_heap_space_summary() override;
 
+  bool contains_null(const oop* p) const override;
+
   void safepoint_synchronize_begin() override;
   void safepoint_synchronize_end() override;
 
diff --git a/src/hotspot/share/gc/z/zContinuation.cpp b/src/hotspot/share/gc/z/zContinuation.cpp
new file mode 100644
index 00000000000..9deb583f813
--- /dev/null
+++ b/src/hotspot/share/gc/z/zContinuation.cpp
@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zBarrier.inline.hpp"
+#include "gc/z/zContinuation.inline.hpp"
+#include "gc/z/zStackChunkGCData.inline.hpp"
+#include "runtime/atomic.hpp"
+
+static zpointer materialize_zpointer(stackChunkOop chunk, void* addr) {
+  volatile uintptr_t* const value_addr = (volatile uintptr_t*)addr;
+
+  // A stack chunk has two modes:
+  //
+  // 1) It's recently allocated and the contents is a copy of the native stack.
+  //    All oops have the format of oops in the stack. That is, they are
+  //    zaddresses, and don't have any colored metadata bits.
+  //
+  // 2) It has lived long enough that the GC needs to visit the oops.
+  //    Before the GC visits the oops, they are converted into zpointers,
+  //    and become colored pointers.
+  //
+  // The load_oop function supports loading oops from chunks in either of the
+  // two modes. It even supports loading oops, while another thread is
+  // converting the chunk to "gc mode" [transition from (1) to (2)]. So, we
+  // load the oop once and perform all checks on that loaded copy.
+
+  // Load once
+  const uintptr_t value = Atomic::load(value_addr);
+
+  if ((value & ~ZPointerAllMetadataMask) == 0) {
+    // Must be null of some sort - either zaddress or zpointer
+    return zpointer::null;
+  }
+
+  const uintptr_t impossible_zaddress_mask = ~((ZAddressHeapBase - 1) | ZAddressHeapBase);
+  if ((value & impossible_zaddress_mask) != 0) {
+    // Must be a zpointer - it has bits forbidden in zaddresses
+    return to_zpointer(value);
+  }
+
+  // Must be zaddress
+  const zaddress_unsafe zaddr = to_zaddress_unsafe(value);
+
+  // A zaddress means that the chunk was recently allocated, and the layout is
+  // that of a native stack. That means that oops are uncolored (zaddress). But
+  // the oops still have an implicit color, saved away in the chunk.
+
+  // Use the implicit color, and create a zpointer that is equivalent with
+  // what we would have written if we where to eagerly create the zpointer
+  // when the stack frames where copied into the chunk.
+  const uintptr_t color = ZStackChunkGCData::color(chunk);
+  return ZAddress::color(zaddr, color);
+}
+
+oop ZContinuation::load_oop(stackChunkOop chunk, void* addr) {
+  // addr could contain either a zpointer or a zaddress
+  const zpointer zptr = materialize_zpointer(chunk, addr);
+
+  // Apply the load barrier, without healing the zaddress/zpointer
+  return to_oop(ZBarrier::load_barrier_on_oop_field_preloaded(nullptr /* p */, zptr));
+}
+
+ZContinuation::ZColorStackOopClosure::ZColorStackOopClosure(stackChunkOop chunk) :
+    _color(ZStackChunkGCData::color(chunk)) {
+}
+
+void ZContinuation::ZColorStackOopClosure::do_oop(oop* p) {
+  // Convert zaddress to zpointer
+  // TODO: Comment why this is safe and non volatile
+  zaddress_unsafe* const p_zaddress_unsafe = (zaddress_unsafe*)p;
+  zpointer* const p_zpointer = (zpointer*)p;
+  *p_zpointer = ZAddress::color(*p_zaddress_unsafe, _color);
+}
+
+void ZContinuation::ZColorStackOopClosure::do_oop(narrowOop* p) {
+  ShouldNotReachHere();
+}
+
+void ZContinuation::ZUncolorStackOopClosure::do_oop(oop* p) {
+  const zpointer ptr = *(volatile zpointer*)p;
+  const zaddress addr = ZPointer::uncolor(ptr);
+  *(volatile zaddress*)p = addr;
+}
+
+void ZContinuation::ZUncolorStackOopClosure::do_oop(narrowOop* p) {
+  ShouldNotReachHere();
+}
diff --git a/src/hotspot/share/gc/z/zContinuation.hpp b/src/hotspot/share/gc/z/zContinuation.hpp
new file mode 100644
index 00000000000..4ae1a29fb2b
--- /dev/null
+++ b/src/hotspot/share/gc/z/zContinuation.hpp
@@ -0,0 +1,57 @@
+/*
+ * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZCONTINUATION_HPP
+#define SHARE_GC_Z_ZCONTINUATION_HPP
+
+#include "memory/allStatic.hpp"
+#include "memory/iterator.hpp"
+#include "oops/oopsHierarchy.hpp"
+
+class OopClosure;
+class ZHeap;
+
+class ZContinuation : public AllStatic {
+public:
+  static bool requires_barriers(const ZHeap* heap, stackChunkOop chunk);
+
+  static oop load_oop(stackChunkOop chunk, void* addr);
+
+  class ZColorStackOopClosure : public OopClosure {
+  private:
+    uintptr_t _color;
+
+  public:
+    ZColorStackOopClosure(stackChunkOop chunk);
+    virtual void do_oop(oop* p) override;
+    virtual void do_oop(narrowOop* p) override;
+  };
+
+  class ZUncolorStackOopClosure : public OopClosure {
+  public:
+    virtual void do_oop(oop* p) override;
+    virtual void do_oop(narrowOop* p) override;
+  };
+};
+
+#endif // SHARE_GC_Z_ZCONTINUATION_HPP
diff --git a/src/hotspot/share/gc/z/zContinuation.inline.hpp b/src/hotspot/share/gc/z/zContinuation.inline.hpp
new file mode 100644
index 00000000000..1e7c09e4da5
--- /dev/null
+++ b/src/hotspot/share/gc/z/zContinuation.inline.hpp
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZCONTINUATION_INLINE_HPP
+#define SHARE_GC_Z_ZCONTINUATION_INLINE_HPP
+
+#include "gc/z/zContinuation.hpp"
+
+#include "classfile/javaClasses.hpp"
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zHeap.inline.hpp"
+#include "gc/z/zStackChunkGCData.inline.hpp"
+#include "oops/oop.inline.hpp"
+
+inline bool ZContinuation::requires_barriers(const ZHeap* heap, stackChunkOop chunk) {
+  if (!heap->is_allocating(to_zaddress(chunk))) {
+    // An object that isn't allocating, is visible from GC tracing. Such
+    // stack chunks require barriers.
+    return true;
+  }
+
+  if (ZStackChunkGCData::color(chunk) != ZPointerStoreGoodMask) {
+    // If a chunk is allocated after a GC started, but before relocate start
+    // we can have an allocating chunk that isn't deeply good. That means that
+    // the contained oops might be bad and require GC barriers.
+    return true;
+  }
+
+  // The chunk is allocating and its pointers are good. This chunk needs no
+  // GC barriers
+  return false;
+}
+
+#endif // SHARE_GC_Z_ZCONTINUATION_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zDebug.gdb b/src/hotspot/share/gc/z/zDebug.gdb
index 210063c3ac3..181da66b9c9 100644
--- a/src/hotspot/share/gc/z/zDebug.gdb
+++ b/src/hotspot/share/gc/z/zDebug.gdb
@@ -14,30 +14,30 @@ define zpo
     set $obj = (oopDesc*)($arg0)
 
     printf "Oop:   0x%016llx\tState: ", (uintptr_t)$obj
-    if ((uintptr_t)$obj & (uintptr_t)ZAddressGoodMask)
+    if ((uintptr_t)$obj & (uintptr_t)ZPointerStoreGoodMask)
         printf "Good "
-        if ((uintptr_t)$obj & (uintptr_t)ZAddressMetadataRemapped)
+        if ((uintptr_t)$obj & (uintptr_t)ZPointerRemapped)
             printf "(Remapped)"
         else
-            if ((uintptr_t)$obj & (uintptr_t)ZAddressMetadataMarked)
-                printf "(Marked)"
+            if ((uintptr_t)$obj & (uintptr_t)ZPointerMarkedOld)
+                printf "(MarkedOld)"
             else
                 printf "(Unknown)"
             end
         end
     else
         printf "Bad "
-        if ((uintptr_t)ZAddressGoodMask & (uintptr_t)ZAddressMetadataMarked)
+        if ((uintptr_t)ZPointerStoreGoodMask & (uintptr_t)ZPointerMarkedOld)
             # Should be marked
-            if ((uintptr_t)$obj & (uintptr_t)ZAddressMetadataRemapped)
+            if ((uintptr_t)$obj & (uintptr_t)ZPointerRemapped)
                 printf "(Not Marked, Remapped)"
             else
                 printf "(Not Marked, Not Remapped)"
             end
         else
-            if ((uintptr_t)ZAddressGoodMask & (uintptr_t)ZAddressMetadataRemapped)
+            if ((uintptr_t)ZPointerStoreGoodMask & (uintptr_t)ZPointerRemapped)
                 # Should be remapped
-                if ((uintptr_t)$obj & (uintptr_t)ZAddressMetadataMarked)
+                if ((uintptr_t)$obj & (uintptr_t)ZPointerMarkedOld)
                     printf "(Marked, Not Remapped)"
                 else
                     printf "(Not Marked, Not Remapped)"
@@ -129,20 +129,99 @@ define zmarked
     end
 end
 
+# For some reason gdb doesn't like ZGeneration::ZPhase::Mark etc.
+# Use hard-coded values instead.
+define z_print_phase
+  if $arg0 == 0
+    printf "Mark"
+  else
+    if $arg0 == 1
+      printf "MarkComplete"
+    else
+      if $arg0 == 2
+        printf "Relocate"
+      else
+	printf "Unknown"
+      end
+    end
+  end
+end
+
+define z_print_generation
+  printf "%u", $arg0->_seqnum
+  printf "/"
+  z_print_phase $arg0->_phase
+end
+
+define zz
+  printf "Old: "
+  z_print_generation ZHeap::_heap->_old
+
+  printf " | "
+
+  printf "Young: "
+  z_print_generation ZHeap::_heap->_young
+
+  printf "\n"
+end
+
 # Print heap information
 define zph
     printf "Heap\n"
-    printf "     GlobalPhase:       %u\n", ZGlobalPhase
-    printf "     GlobalSeqNum:      %u\n", ZGlobalSeqNum
+    printf "     Young Phase:       %u\n", ZHeap::_heap->_young->_phase
+    printf "     Old Phase:         %u\n", ZHeap::_heap->_old->_phase
+    printf "     Young SeqNum:      %u\n", ZHeap::_heap->_young->_seqnum
+    printf "     Old SeqNum:        %u\n", ZHeap::_heap->_old->_seqnum
     printf "     Offset Max:        %-15llu (0x%llx)\n", ZAddressOffsetMax, ZAddressOffsetMax
     printf "     Page Size Small:   %-15llu (0x%llx)\n", ZPageSizeSmall, ZPageSizeSmall
     printf "     Page Size Medium:  %-15llu (0x%llx)\n", ZPageSizeMedium, ZPageSizeMedium
     printf "Metadata Bits\n"
-    printf "     Good:              0x%016llx\n", ZAddressGoodMask
-    printf "     Bad:               0x%016llx\n", ZAddressBadMask
-    printf "     WeakBad:           0x%016llx\n", ZAddressWeakBadMask
-    printf "     Marked:            0x%016llx\n", ZAddressMetadataMarked
-    printf "     Remapped:          0x%016llx\n", ZAddressMetadataRemapped
+    printf "     Good:              0x%016llx\n", ZPointerStoreGoodMask
+    printf "     Bad:               0x%016llx\n", ZPointerStoreBadMask
+    printf "     MarkedYoung:       0x%016llx\n", ZPointerMarkedYoung
+    printf "     MarkedOld:         0x%016llx\n", ZPointerMarkedOld
+    printf "     Remapped:          0x%016llx\n", ZPointerRemapped
+end
+
+define print_bits
+  set $value=$arg0
+  set $bits=$arg1
+
+  set $bit=0
+    while ($bit < $bits)
+	set $bit_pos = (1ull << ($bits - 1 - $bit))
+	printf "%d", ($arg0 & $bit_pos) != 0
+  	set $bit = $bit + 1
+  end
+
+  printf " <%lX>", $value
+end
+
+define print_bits8
+  print_bits $arg0 8
+end
+
+define print_s_bits8
+  printf $arg0
+  print_bits8 $arg1
+end
+
+# Print metadata information
+define zpm
+    printf          "Metadata Load Bits  "
+    print_s_bits8 "\n     Mask:          " ZPointerLoadMetadataMask
+    print_s_bits8 "\n     Good:          " ZPointerLoadGoodMask
+    print_s_bits8 "\n     Remapped:      " ZPointerRemapped
+    print_s_bits8 "\n     Bad:           " ZPointerLoadBadMask
+    printf        "\n                    "
+    printf        "\nMetadata Store Bits "
+    print_s_bits8 "\n     Mask:          " ZPointerStoreMetadataMask
+    print_s_bits8 "\n     Good:          " ZPointerStoreGoodMask
+    print_s_bits8 "\n     Bad:           " ZPointerStoreBadMask
+    print_s_bits8 "\n     MarkedYoung:   " ZPointerMarkedYoung
+    print_s_bits8 "\n     MarkedOld:     " ZPointerMarkedOld
+    print_s_bits8 "\n     Finalizable:   " ZPointerFinalizable
+    printf        "\n"
 end
 
 # End of file
diff --git a/src/hotspot/share/gc/z/zDirector.cpp b/src/hotspot/share/gc/z/zDirector.cpp
index a9668a6a77f..ac2c0ccfe4f 100644
--- a/src/hotspot/share/gc/z/zDirector.cpp
+++ b/src/hotspot/share/gc/z/zDirector.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,89 +23,77 @@
 
 #include "precompiled.hpp"
 #include "gc/shared/gc_globals.hpp"
+#include "gc/z/zCollectedHeap.hpp"
 #include "gc/z/zDirector.hpp"
 #include "gc/z/zDriver.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zHeuristics.hpp"
+#include "gc/z/zLock.inline.hpp"
 #include "gc/z/zStat.hpp"
 #include "logging/log.hpp"
 
-constexpr double one_in_1000 = 3.290527;
-constexpr double sample_interval = 1.0 / ZStatAllocRate::sample_hz;
+ZDirector* ZDirector::_director;
 
-ZDirector::ZDirector(ZDriver* driver) :
-    _driver(driver),
-    _metronome(ZStatAllocRate::sample_hz) {
+constexpr double one_in_1000 = 3.290527;
+
+struct ZWorkerResizeStats {
+  bool   _is_active;
+  double _serial_gc_time_passed;
+  double _parallel_gc_time_passed;
+  uint   _nworkers_current;
+};
+
+struct ZDirectorHeapStats {
+  size_t _soft_max_heap_size;
+  size_t _used;
+  uint   _total_collections;
+};
+
+struct ZDirectorGenerationGeneralStats {
+  size_t _used;
+  uint   _total_collections_at_start;
+};
+
+struct ZDirectorGenerationStats {
+  ZStatCycleStats                 _cycle;
+  ZStatWorkersStats               _workers;
+  ZWorkerResizeStats              _resize;
+  ZStatHeapStats                  _stat_heap;
+  ZDirectorGenerationGeneralStats _general;
+};
+
+struct ZDirectorStats {
+  ZStatMutatorAllocRateStats _mutator_alloc_rate;
+  ZDirectorHeapStats         _heap;
+  ZDirectorGenerationStats   _young_stats;
+  ZDirectorGenerationStats   _old_stats;
+};
+
+ZDirector::ZDirector() :
+    _monitor(),
+    _stopped(false) {
+  _director = this;
   set_name("ZDirector");
   create_and_start();
 }
 
-static void sample_allocation_rate() {
-  // Sample allocation rate. This is needed by rule_allocation_rate()
-  // below to estimate the time we have until we run out of memory.
-  const double bytes_per_second = ZStatAllocRate::sample_and_reset();
+// Minor GC rules
 
-  log_debug(gc, alloc)("Allocation Rate: %.1fMB/s, Predicted: %.1fMB/s, Avg: %.1f(+/-%.1f)MB/s",
-                       bytes_per_second / M,
-                       ZStatAllocRate::predict() / M,
-                       ZStatAllocRate::avg() / M,
-                       ZStatAllocRate::sd() / M);
-}
-
-static ZDriverRequest rule_allocation_stall() {
-  // Perform GC if we've observed at least one allocation stall since
-  // the last GC started.
-  if (!ZHeap::heap()->has_alloc_stalled()) {
-    return GCCause::_no_gc;
-  }
-
-  log_debug(gc, director)("Rule: Allocation Stall Observed");
-
-  return GCCause::_z_allocation_stall;
-}
-
-static ZDriverRequest rule_warmup() {
-  if (ZStatCycle::is_warm()) {
+static bool rule_minor_timer(const ZDirectorStats& stats) {
+  if (ZCollectionIntervalMinor <= 0) {
     // Rule disabled
-    return GCCause::_no_gc;
-  }
-
-  // Perform GC if heap usage passes 10/20/30% and no other GC has been
-  // performed yet. This allows us to get some early samples of the GC
-  // duration, which is needed by the other rules.
-  const size_t soft_max_capacity = ZHeap::heap()->soft_max_capacity();
-  const size_t used = ZHeap::heap()->used();
-  const double used_threshold_percent = (ZStatCycle::nwarmup_cycles() + 1) * 0.1;
-  const size_t used_threshold = soft_max_capacity * used_threshold_percent;
-
-  log_debug(gc, director)("Rule: Warmup %.0f%%, Used: " SIZE_FORMAT "MB, UsedThreshold: " SIZE_FORMAT "MB",
-                          used_threshold_percent * 100, used / M, used_threshold / M);
-
-  if (used < used_threshold) {
-    return GCCause::_no_gc;
-  }
-
-  return GCCause::_z_warmup;
-}
-
-static ZDriverRequest rule_timer() {
-  if (ZCollectionInterval <= 0) {
-    // Rule disabled
-    return GCCause::_no_gc;
+    return false;
   }
 
   // Perform GC if timer has expired.
-  const double time_since_last_gc = ZStatCycle::time_since_last();
-  const double time_until_gc = ZCollectionInterval - time_since_last_gc;
+  const double time_since_last_gc = stats._young_stats._cycle._time_since_last;
+  const double time_until_gc = ZCollectionIntervalMinor - time_since_last_gc;
 
-  log_debug(gc, director)("Rule: Timer, Interval: %.3fs, TimeUntilGC: %.3fs",
-                          ZCollectionInterval, time_until_gc);
+  log_debug(gc, director)("Rule Minor: Timer, Interval: %.3fs, TimeUntilGC: %.3fs",
+                          ZCollectionIntervalMinor, time_until_gc);
 
-  if (time_until_gc > 0) {
-    return GCCause::_no_gc;
-  }
-
-  return GCCause::_z_timer;
+  return time_until_gc <= 0;
 }
 
 static double estimated_gc_workers(double serial_gc_time, double parallelizable_gc_time, double time_until_deadline) {
@@ -113,42 +101,29 @@ static double estimated_gc_workers(double serial_gc_time, double parallelizable_
   return parallelizable_gc_time / parallelizable_time_until_deadline;
 }
 
-static uint discrete_gc_workers(double gc_workers) {
-  return clamp<uint>(ceil(gc_workers), 1, ConcGCThreads);
+static uint discrete_young_gc_workers(double gc_workers) {
+  return clamp<uint>(ceil(gc_workers), 1, ZYoungGCThreads);
 }
 
-static double select_gc_workers(double serial_gc_time, double parallelizable_gc_time, double alloc_rate_sd_percent, double time_until_oom) {
+static double select_young_gc_workers(const ZDirectorStats& stats, double serial_gc_time, double parallelizable_gc_time, double alloc_rate_sd_percent, double time_until_oom) {
   // Use all workers until we're warm
-  if (!ZStatCycle::is_warm()) {
-    const double not_warm_gc_workers = ConcGCThreads;
-    log_debug(gc, director)("Select GC Workers (Not Warm), GCWorkers: %.3f", not_warm_gc_workers);
+  if (!stats._old_stats._cycle._is_warm) {
+    const double not_warm_gc_workers = ZYoungGCThreads;
+    log_debug(gc, director)("Select Minor GC Workers (Not Warm), GCWorkers: %.3f", not_warm_gc_workers);
     return not_warm_gc_workers;
   }
 
-  // Calculate number of GC workers needed to avoid a long GC cycle and to avoid OOM.
-  const double avoid_long_gc_workers = estimated_gc_workers(serial_gc_time, parallelizable_gc_time, 10 /* seconds */);
-  const double avoid_oom_gc_workers = estimated_gc_workers(serial_gc_time, parallelizable_gc_time, time_until_oom);
+  // Calculate number of GC workers needed to avoid OOM.
+  const double gc_workers = estimated_gc_workers(serial_gc_time, parallelizable_gc_time, time_until_oom);
+  const uint actual_gc_workers = discrete_young_gc_workers(gc_workers);
+  const double last_gc_workers = stats._young_stats._cycle._last_active_workers;
 
-  const double gc_workers = MAX2(avoid_long_gc_workers, avoid_oom_gc_workers);
-  const uint actual_gc_workers = discrete_gc_workers(gc_workers);
-  const uint last_gc_workers = ZStatCycle::last_active_workers();
-
-  // More than 15% division from the average is considered unsteady
-  if (alloc_rate_sd_percent >= 0.15) {
-    const double half_gc_workers = ConcGCThreads / 2.0;
-    const double unsteady_gc_workers = MAX3<double>(gc_workers, last_gc_workers, half_gc_workers);
-    log_debug(gc, director)("Select GC Workers (Unsteady), "
-                            "AvoidLongGCWorkers: %.3f, AvoidOOMGCWorkers: %.3f, LastGCWorkers: %.3f, HalfGCWorkers: %.3f, GCWorkers: %.3f",
-                            avoid_long_gc_workers, avoid_oom_gc_workers, (double)last_gc_workers, half_gc_workers, unsteady_gc_workers);
-    return unsteady_gc_workers;
-  }
-
-  if (actual_gc_workers < last_gc_workers) {
+  if ((double)actual_gc_workers < last_gc_workers) {
     // Before decreasing number of GC workers compared to the previous GC cycle, check if the
     // next GC cycle will need to increase it again. If so, use the same number of GC workers
     // that will be needed in the next cycle.
     const double gc_duration_delta = (parallelizable_gc_time / actual_gc_workers) - (parallelizable_gc_time / last_gc_workers);
-    const double additional_time_for_allocations = ZStatCycle::time_since_last() - gc_duration_delta - sample_interval;
+    const double additional_time_for_allocations = stats._young_stats._cycle._time_since_last - gc_duration_delta;
     const double next_time_until_oom = time_until_oom + additional_time_for_allocations;
     const double next_avoid_oom_gc_workers = estimated_gc_workers(serial_gc_time, parallelizable_gc_time, next_time_until_oom);
 
@@ -156,29 +131,32 @@ static double select_gc_workers(double serial_gc_time, double parallelizable_gc_
     const double next_gc_workers = next_avoid_oom_gc_workers + 0.5;
     const double try_lowering_gc_workers = clamp<double>(next_gc_workers, actual_gc_workers, last_gc_workers);
 
-    log_debug(gc, director)("Select GC Workers (Try Lowering), "
-                           "AvoidLongGCWorkers: %.3f, AvoidOOMGCWorkers: %.3f, NextAvoidOOMGCWorkers: %.3f, LastGCWorkers: %.3f, GCWorkers: %.3f",
-                            avoid_long_gc_workers, avoid_oom_gc_workers, next_avoid_oom_gc_workers, (double)last_gc_workers, try_lowering_gc_workers);
+    log_debug(gc, director)("Select Minor GC Workers (Try Lowering), "
+                            "AvoidOOMGCWorkers: %.3f, NextAvoidOOMGCWorkers: %.3f, LastGCWorkers: %.3f, GCWorkers: %.3f",
+                            gc_workers, next_avoid_oom_gc_workers, last_gc_workers, try_lowering_gc_workers);
     return try_lowering_gc_workers;
   }
 
-  log_debug(gc, director)("Select GC Workers (Normal), "
-                         "AvoidLongGCWorkers: %.3f, AvoidOOMGCWorkers: %.3f, LastGCWorkers: %.3f, GCWorkers: %.3f",
-                         avoid_long_gc_workers, avoid_oom_gc_workers, (double)last_gc_workers, gc_workers);
+  log_debug(gc, director)("Select Minor GC Workers (Normal), "
+                          "AvoidOOMGCWorkers: %.3f, LastGCWorkers: %.3f, GCWorkers: %.3f",
+                          gc_workers, last_gc_workers, gc_workers);
   return gc_workers;
 }
 
-ZDriverRequest rule_allocation_rate_dynamic() {
-  if (!ZStatCycle::is_time_trustable()) {
+ZDriverRequest rule_minor_allocation_rate_dynamic(const ZDirectorStats& stats,
+                                                  double serial_gc_time_passed,
+                                                  double parallel_gc_time_passed,
+                                                  bool conservative_alloc_rate,
+                                                  size_t capacity) {
+  if (!stats._old_stats._cycle._is_time_trustable) {
     // Rule disabled
-    return GCCause::_no_gc;
+    return ZDriverRequest(GCCause::_no_gc, ZYoungGCThreads, 0);
   }
 
   // Calculate amount of free memory available. Note that we take the
   // relocation headroom into account to avoid in-place relocation.
-  const size_t soft_max_capacity = ZHeap::heap()->soft_max_capacity();
-  const size_t used = ZHeap::heap()->used();
-  const size_t free_including_headroom = soft_max_capacity - MIN2(soft_max_capacity, used);
+  const size_t used = stats._heap._used;
+  const size_t free_including_headroom = capacity - MIN2(capacity, used);
   const size_t free = free_including_headroom - MIN2(free_including_headroom, ZHeuristics::relocation_headroom());
 
   // Calculate time until OOM given the max allocation rate and the amount
@@ -187,36 +165,35 @@ ZDriverRequest rule_allocation_rate_dynamic() {
   // phase changes in the allocate rate. We then add ~3.3 sigma to account for
   // the allocation rate variance, which means the probability is 1 in 1000
   // that a sample is outside of the confidence interval.
-  const double alloc_rate_predict = ZStatAllocRate::predict();
-  const double alloc_rate_avg = ZStatAllocRate::avg();
-  const double alloc_rate_sd = ZStatAllocRate::sd();
+  const ZStatMutatorAllocRateStats alloc_rate_stats = stats._mutator_alloc_rate;
+  const double alloc_rate_predict = alloc_rate_stats._predict;
+  const double alloc_rate_avg = alloc_rate_stats._avg;
+  const double alloc_rate_sd = alloc_rate_stats._sd;
   const double alloc_rate_sd_percent = alloc_rate_sd / (alloc_rate_avg + 1.0);
-  const double alloc_rate = (MAX2(alloc_rate_predict, alloc_rate_avg) * ZAllocationSpikeTolerance) + (alloc_rate_sd * one_in_1000) + 1.0;
+  const double alloc_rate_conservative = (MAX2(alloc_rate_predict, alloc_rate_avg) * ZAllocationSpikeTolerance) + (alloc_rate_sd * one_in_1000) + 1.0;
+  const double alloc_rate = conservative_alloc_rate ? alloc_rate_conservative : alloc_rate_stats._avg;
   const double time_until_oom = (free / alloc_rate) / (1.0 + alloc_rate_sd_percent);
 
   // Calculate max serial/parallel times of a GC cycle. The times are
   // moving averages, we add ~3.3 sigma to account for the variance.
-  const double serial_gc_time = ZStatCycle::serial_time().davg() + (ZStatCycle::serial_time().dsd() * one_in_1000);
-  const double parallelizable_gc_time = ZStatCycle::parallelizable_time().davg() + (ZStatCycle::parallelizable_time().dsd() * one_in_1000);
+  const double serial_gc_time = fabsd(stats._young_stats._cycle._avg_serial_time + (stats._young_stats._cycle._sd_serial_time * one_in_1000) - serial_gc_time_passed);
+  const double parallelizable_gc_time = fabsd(stats._young_stats._cycle._avg_parallelizable_time + (stats._young_stats._cycle._sd_parallelizable_time * one_in_1000) - parallel_gc_time_passed);
 
   // Calculate number of GC workers needed to avoid OOM.
-  const double gc_workers = select_gc_workers(serial_gc_time, parallelizable_gc_time, alloc_rate_sd_percent, time_until_oom);
+  const double gc_workers = select_young_gc_workers(stats, serial_gc_time, parallelizable_gc_time, alloc_rate_sd_percent, time_until_oom);
 
   // Convert to a discrete number of GC workers within limits.
-  const uint actual_gc_workers = discrete_gc_workers(gc_workers);
+  const uint actual_gc_workers = discrete_young_gc_workers(gc_workers);
 
   // Calculate GC duration given number of GC workers needed.
   const double actual_gc_duration = serial_gc_time + (parallelizable_gc_time / actual_gc_workers);
-  const uint last_gc_workers = ZStatCycle::last_active_workers();
 
   // Calculate time until GC given the time until OOM and GC duration.
-  // We also subtract the sample interval, so that we don't overshoot the
-  // target time and end up starting the GC too late in the next interval.
-  const double time_until_gc = time_until_oom - actual_gc_duration - sample_interval;
+  const double time_until_gc = time_until_oom - actual_gc_duration;
 
-  log_debug(gc, director)("Rule: Allocation Rate (Dynamic GC Workers), "
+  log_debug(gc, director)("Rule Minor: Allocation Rate (Dynamic GC Workers), "
                           "MaxAllocRate: %.1fMB/s (+/-%.1f%%), Free: " SIZE_FORMAT "MB, GCCPUTime: %.3f, "
-                          "GCDuration: %.3fs, TimeUntilOOM: %.3fs, TimeUntilGC: %.3fs, GCWorkers: %u -> %u",
+                          "GCDuration: %.3fs, TimeUntilOOM: %.3fs, TimeUntilGC: %.3fs, GCWorkers: %u",
                           alloc_rate / M,
                           alloc_rate_sd_percent * 100,
                           free / M,
@@ -224,20 +201,53 @@ ZDriverRequest rule_allocation_rate_dynamic() {
                           serial_gc_time + (parallelizable_gc_time / actual_gc_workers),
                           time_until_oom,
                           time_until_gc,
-                          last_gc_workers,
                           actual_gc_workers);
 
-  if (actual_gc_workers <= last_gc_workers && time_until_gc > 0) {
-    return ZDriverRequest(GCCause::_no_gc, actual_gc_workers);
+  // Bail out if we are not "close" to needing the GC to start yet, where
+  // close is 5% of the time left until OOM. If we don't check that we
+  // are "close", then the heuristics instead add more threads and we
+  // end up not triggering GCs until we have the max number of threads.
+  if (time_until_gc > time_until_oom * 0.05) {
+    return ZDriverRequest(GCCause::_no_gc, actual_gc_workers, 0);
   }
 
-  return ZDriverRequest(GCCause::_z_allocation_rate, actual_gc_workers);
+  return ZDriverRequest(GCCause::_z_allocation_rate, actual_gc_workers, 0);
 }
 
-static ZDriverRequest rule_allocation_rate_static() {
-  if (!ZStatCycle::is_time_trustable()) {
+ZDriverRequest rule_soft_minor_allocation_rate_dynamic(const ZDirectorStats& stats,
+                                                       double serial_gc_time_passed,
+                                                       double parallel_gc_time_passed) {
+    return rule_minor_allocation_rate_dynamic(stats,
+                                              0.0 /* serial_gc_time_passed */,
+                                              0.0 /* parallel_gc_time_passed */,
+                                              false /* conservative_alloc_rate */,
+                                              stats._heap._soft_max_heap_size /* capacity */);
+}
+
+ZDriverRequest rule_semi_hard_minor_allocation_rate_dynamic(const ZDirectorStats& stats,
+                                                            double serial_gc_time_passed,
+                                                            double parallel_gc_time_passed) {
+  return rule_minor_allocation_rate_dynamic(stats,
+                                            0.0 /* serial_gc_time_passed */,
+                                            0.0 /* parallel_gc_time_passed */,
+                                            false /* conservative_alloc_rate */,
+                                            ZHeap::heap()->max_capacity() /* capacity */);
+}
+
+ZDriverRequest rule_hard_minor_allocation_rate_dynamic(const ZDirectorStats& stats,
+                                                       double serial_gc_time_passed,
+                                                       double parallel_gc_time_passed) {
+  return rule_minor_allocation_rate_dynamic(stats,
+                                            0.0 /* serial_gc_time_passed */,
+                                            0.0 /* parallel_gc_time_passed */,
+                                            true /* conservative_alloc_rate */,
+                                            ZHeap::heap()->max_capacity() /* capacity */);
+}
+
+static bool rule_minor_allocation_rate_static(const ZDirectorStats& stats) {
+  if (!stats._old_stats._cycle._is_time_trustable) {
     // Rule disabled
-    return GCCause::_no_gc;
+    return false;
   }
 
   // Perform GC if the estimated max allocation rate indicates that we
@@ -248,8 +258,8 @@ static ZDriverRequest rule_allocation_rate_static() {
 
   // Calculate amount of free memory available. Note that we take the
   // relocation headroom into account to avoid in-place relocation.
-  const size_t soft_max_capacity = ZHeap::heap()->soft_max_capacity();
-  const size_t used = ZHeap::heap()->used();
+  const size_t soft_max_capacity = stats._heap._soft_max_heap_size;
+  const size_t used = stats._heap._used;
   const size_t free_including_headroom = soft_max_capacity - MIN2(soft_max_capacity, used);
   const size_t free = free_including_headroom - MIN2(free_including_headroom, ZHeuristics::relocation_headroom());
 
@@ -259,69 +269,286 @@ static ZDriverRequest rule_allocation_rate_static() {
   // phase changes in the allocate rate. We then add ~3.3 sigma to account for
   // the allocation rate variance, which means the probability is 1 in 1000
   // that a sample is outside of the confidence interval.
-  const double max_alloc_rate = (ZStatAllocRate::avg() * ZAllocationSpikeTolerance) + (ZStatAllocRate::sd() * one_in_1000);
+  const ZStatMutatorAllocRateStats alloc_rate_stats = stats._mutator_alloc_rate;
+  const double max_alloc_rate = (alloc_rate_stats._avg * ZAllocationSpikeTolerance) + (alloc_rate_stats._sd * one_in_1000);
   const double time_until_oom = free / (max_alloc_rate + 1.0); // Plus 1.0B/s to avoid division by zero
 
   // Calculate max serial/parallel times of a GC cycle. The times are
   // moving averages, we add ~3.3 sigma to account for the variance.
-  const double serial_gc_time = ZStatCycle::serial_time().davg() + (ZStatCycle::serial_time().dsd() * one_in_1000);
-  const double parallelizable_gc_time = ZStatCycle::parallelizable_time().davg() + (ZStatCycle::parallelizable_time().dsd() * one_in_1000);
+  const double serial_gc_time = stats._young_stats._cycle._avg_serial_time + (stats._young_stats._cycle._sd_serial_time * one_in_1000);
+  const double parallelizable_gc_time = stats._young_stats._cycle._avg_parallelizable_time + (stats._young_stats._cycle._sd_parallelizable_time * one_in_1000);
 
   // Calculate GC duration given number of GC workers needed.
-  const double gc_duration = serial_gc_time + (parallelizable_gc_time / ConcGCThreads);
+  const double gc_duration = serial_gc_time + (parallelizable_gc_time / ZYoungGCThreads);
 
   // Calculate time until GC given the time until OOM and max duration of GC.
   // We also deduct the sample interval, so that we don't overshoot the target
   // time and end up starting the GC too late in the next interval.
-  const double time_until_gc = time_until_oom - gc_duration - sample_interval;
+  const double time_until_gc = time_until_oom - gc_duration;
 
-  log_debug(gc, director)("Rule: Allocation Rate (Static GC Workers), MaxAllocRate: %.1fMB/s, Free: " SIZE_FORMAT "MB, GCDuration: %.3fs, TimeUntilGC: %.3fs",
+  log_debug(gc, director)("Rule Minor: Allocation Rate (Static GC Workers), MaxAllocRate: %.1fMB/s, Free: " SIZE_FORMAT "MB, GCDuration: %.3fs, TimeUntilGC: %.3fs",
                           max_alloc_rate / M, free / M, gc_duration, time_until_gc);
 
-  if (time_until_gc > 0) {
-    return GCCause::_no_gc;
-  }
-
-  return GCCause::_z_allocation_rate;
+  return time_until_gc <= 0;
 }
 
-static ZDriverRequest rule_allocation_rate() {
-  if (UseDynamicNumberOfGCThreads) {
-    return rule_allocation_rate_dynamic();
-  } else {
-    return rule_allocation_rate_static();
-  }
+static bool is_young_small(const ZDirectorStats& stats) {
+  // Calculate amount of freeable memory available.
+  const size_t soft_max_capacity = stats._heap._soft_max_heap_size;
+  const size_t young_used = stats._young_stats._general._used;
+
+  const double young_used_percent = percent_of(young_used, soft_max_capacity);
+
+  // If the freeable memory isn't even 5% of the heap, we can't expect to free up
+  // all that much memory, so let's not even try - it will likely be a wasted effort
+  // that takes away CPU power to the hopefullt more profitable major colelction.
+  return young_used_percent <= 5.0;
 }
 
-static ZDriverRequest rule_high_usage() {
-  // Perform GC if the amount of free memory is 5% or less. This is a preventive
-  // meassure in the case where the application has a very low allocation rate,
-  // such that the allocation rate rule doesn't trigger, but the amount of free
-  // memory is still slowly but surely heading towards zero. In this situation,
-  // we start a GC cycle to avoid a potential allocation stall later.
-
+template <typename PrintFn = void(*)(size_t, double)>
+static bool is_high_usage(const ZDirectorStats& stats, PrintFn* print_function = nullptr) {
   // Calculate amount of free memory available. Note that we take the
   // relocation headroom into account to avoid in-place relocation.
-  const size_t soft_max_capacity = ZHeap::heap()->soft_max_capacity();
-  const size_t used = ZHeap::heap()->used();
+  const size_t soft_max_capacity = stats._heap._soft_max_heap_size;
+  const size_t used = stats._heap._used;
   const size_t free_including_headroom = soft_max_capacity - MIN2(soft_max_capacity, used);
   const size_t free = free_including_headroom - MIN2(free_including_headroom, ZHeuristics::relocation_headroom());
   const double free_percent = percent_of(free, soft_max_capacity);
 
-  log_debug(gc, director)("Rule: High Usage, Free: " SIZE_FORMAT "MB(%.1f%%)",
-                          free / M, free_percent);
-
-  if (free_percent > 5.0) {
-    return GCCause::_no_gc;
+  if (print_function != nullptr) {
+    (*print_function)(free, free_percent);
   }
 
-  return GCCause::_z_high_usage;
+  // The heap has high usage if there is less than 5% free memory left
+  return free_percent <= 5.0;
 }
 
-static ZDriverRequest rule_proactive() {
-  if (!ZProactive || !ZStatCycle::is_warm()) {
+static bool is_major_urgent(const ZDirectorStats& stats) {
+  return is_young_small(stats) && is_high_usage(stats);
+}
+
+static bool rule_minor_allocation_rate(const ZDirectorStats& stats) {
+  if (ZCollectionIntervalOnly) {
     // Rule disabled
-    return GCCause::_no_gc;
+    return false;
+  }
+
+  if (ZHeap::heap()->is_alloc_stalling_for_old()) {
+    // Don't collect young if we have threads stalled waiting for an old collection
+    return false;
+  }
+
+  if (is_young_small(stats)) {
+    return false;
+  }
+
+  if (UseDynamicNumberOfGCThreads) {
+    if (rule_soft_minor_allocation_rate_dynamic(stats,
+                                                0.0 /* serial_gc_time_passed */,
+                                                0.0 /* parallel_gc_time_passed */).cause() != GCCause::_no_gc) {
+      return true;
+    }
+
+    if (rule_hard_minor_allocation_rate_dynamic(stats,
+                                                0.0 /* serial_gc_time_passed */,
+                                                0.0 /* parallel_gc_time_passed */).cause() != GCCause::_no_gc) {
+      return true;
+    }
+
+    return false;
+  }
+
+  return rule_minor_allocation_rate_static(stats);
+}
+
+static bool rule_minor_high_usage(const ZDirectorStats& stats) {
+  if (ZCollectionIntervalOnly) {
+    // Rule disabled
+    return false;
+  }
+
+  if (is_young_small(stats)) {
+    return false;
+  }
+
+  // Perform GC if the amount of free memory is small. This is a preventive
+  // measure in the case where the application has a very low allocation rate,
+  // such that the allocation rate rule doesn't trigger, but the amount of free
+  // memory is still slowly but surely heading towards zero. In this situation,
+  // we start a GC cycle to avoid a potential allocation stall later.
+
+  const size_t soft_max_capacity = stats._heap._soft_max_heap_size;
+  const size_t used = stats._heap._used;
+  const size_t free_including_headroom = soft_max_capacity - MIN2(soft_max_capacity, used);
+  const size_t free = free_including_headroom - MIN2(free_including_headroom, ZHeuristics::relocation_headroom());
+  const double free_percent = percent_of(free, soft_max_capacity);
+
+  auto print_function = [&](size_t free, double free_percent) {
+    log_debug(gc, director)("Rule Minor: High Usage, Free: " SIZE_FORMAT "MB(%.1f%%)",
+                            free / M, free_percent);
+  };
+
+  return is_high_usage(stats, &print_function);
+}
+
+// Major GC rules
+
+static bool rule_major_timer(const ZDirectorStats& stats) {
+  if (ZCollectionIntervalMajor <= 0) {
+    // Rule disabled
+    return false;
+  }
+
+  // Perform GC if timer has expired.
+  const double time_since_last_gc = stats._old_stats._cycle._time_since_last;
+  const double time_until_gc = ZCollectionIntervalMajor - time_since_last_gc;
+
+  log_debug(gc, director)("Rule Major: Timer, Interval: %.3fs, TimeUntilGC: %.3fs",
+                          ZCollectionIntervalMajor, time_until_gc);
+
+  return time_until_gc <= 0;
+}
+
+static bool rule_major_warmup(const ZDirectorStats& stats) {
+  if (ZCollectionIntervalOnly) {
+    // Rule disabled
+    return false;
+  }
+
+  if (stats._old_stats._cycle._is_warm) {
+    // Rule disabled
+    return false;
+  }
+
+  // Perform GC if heap usage passes 10/20/30% and no other GC has been
+  // performed yet. This allows us to get some early samples of the GC
+  // duration, which is needed by the other rules.
+  const size_t soft_max_capacity = stats._heap._soft_max_heap_size;
+  const size_t used = stats._heap._used;
+  const double used_threshold_percent = (stats._old_stats._cycle._nwarmup_cycles + 1) * 0.1;
+  const size_t used_threshold = soft_max_capacity * used_threshold_percent;
+
+  log_debug(gc, director)("Rule Major: Warmup %.0f%%, Used: " SIZE_FORMAT "MB, UsedThreshold: " SIZE_FORMAT "MB",
+                          used_threshold_percent * 100, used / M, used_threshold / M);
+
+  return used >= used_threshold;
+}
+
+static double gc_time(ZDirectorGenerationStats generation_stats) {
+  // Calculate max serial/parallel times of a generation GC cycle. The times are
+  // moving averages, we add ~3.3 sigma to account for the variance.
+  const double serial_gc_time = generation_stats._cycle._avg_serial_time + (generation_stats._cycle._sd_serial_time * one_in_1000);
+  const double parallelizable_gc_time = generation_stats._cycle._avg_parallelizable_time + (generation_stats._cycle._sd_parallelizable_time * one_in_1000);
+
+  // Calculate young GC time and duration given number of GC workers needed.
+  return serial_gc_time + parallelizable_gc_time;
+}
+
+static double calculate_extra_young_gc_time(const ZDirectorStats& stats) {
+  if (!stats._old_stats._cycle._is_time_trustable) {
+    return 0.0;
+  }
+
+  // Calculate amount of free memory available. Note that we take the
+  // relocation headroom into account to avoid in-place relocation.
+  const size_t old_used = stats._old_stats._general._used;
+  const size_t old_live = stats._old_stats._stat_heap._live_at_mark_end;
+  const size_t old_garbage = old_used - old_live;
+
+  const double young_gc_time = gc_time(stats._young_stats);
+
+  // Calculate how much memory young collections are predicted to free.
+  const size_t reclaimed_per_young_gc = stats._young_stats._stat_heap._reclaimed_avg;
+
+  // Calculate current YC time and predicted YC time after an old collection.
+  const double current_young_gc_time_per_bytes_freed = double(young_gc_time) / double(reclaimed_per_young_gc);
+  const double potential_young_gc_time_per_bytes_freed = double(young_gc_time) / double(reclaimed_per_young_gc + old_garbage);
+
+  // Calculate extra time per young collection inflicted by *not* doing an
+  // old collection that frees up memory in the old generation.
+  const double extra_young_gc_time_per_bytes_freed = current_young_gc_time_per_bytes_freed - potential_young_gc_time_per_bytes_freed;
+  const double extra_young_gc_time = extra_young_gc_time_per_bytes_freed * (reclaimed_per_young_gc + old_garbage);
+
+  return extra_young_gc_time;
+}
+
+static bool rule_major_allocation_rate(const ZDirectorStats& stats) {
+  if (!stats._old_stats._cycle._is_time_trustable) {
+    // Rule disabled
+    return false;
+  }
+
+  // Calculate GC time.
+  const double old_gc_time = gc_time(stats._old_stats);
+  const double young_gc_time = gc_time(stats._young_stats);
+
+  // Calculate how much memory collections are predicted to free.
+  const size_t reclaimed_per_young_gc = stats._young_stats._stat_heap._reclaimed_avg;
+  const size_t reclaimed_per_old_gc = stats._old_stats._stat_heap._reclaimed_avg;
+
+  // Calculate the GC cost for each reclaimed byte
+  const double current_young_gc_time_per_bytes_freed = double(young_gc_time) / double(reclaimed_per_young_gc);
+  const double current_old_gc_time_per_bytes_freed = double(old_gc_time) / double(reclaimed_per_old_gc);
+
+  // Calculate extra time per young collection inflicted by *not* doing an
+  // old collection that frees up memory in the old generation.
+  const double extra_young_gc_time = calculate_extra_young_gc_time(stats);
+
+  // Doing an old collection makes subsequent young collections more efficient.
+  // Calculate the number of young collections ahead that we will try to amortize
+  // the cost of doing an old collection for.
+  const int lookahead = stats._heap._total_collections - stats._old_stats._general._total_collections_at_start;
+
+  // Calculate extra young collection overhead predicted for a number of future
+  // young collections, due to not freeing up memory in the old generation.
+  const double extra_young_gc_time_for_lookahead = extra_young_gc_time * lookahead;
+
+  log_debug(gc, director)("Rule Major: Allocation Rate, ExtraYoungGCTime: %.3fs, OldGCTime: %.3fs, Lookahead: %d, ExtraYoungGCTimeForLookahead: %.3fs",
+                          extra_young_gc_time, old_gc_time, lookahead, extra_young_gc_time_for_lookahead);
+
+  // If we continue doing as many minor collections as we already did since the
+  // last major collection (N), without doing a major collection, then the minor
+  // GC effort of freeing up memory for another N cycles, plus the effort of doing,
+  // a major GC combined, is lower compared to the extra GC overhead per minor
+  // collection, freeing an equal amount of memory, at a higher GC frequency.
+  // In other words, the cost for minor collections of not doing a major collection
+  // will seemingly be greater than the cost of doing a major collection and getting
+  // cheaper minor collections for a time to come.
+  const bool can_amortize_time_cost = extra_young_gc_time_for_lookahead > old_gc_time;
+
+  // If the garbage is cheaper to reap in the old generation, then it makes sense
+  // to upgrade minor collections to major collections.
+  const bool old_garbage_is_cheaper = current_old_gc_time_per_bytes_freed < current_young_gc_time_per_bytes_freed;
+
+  return can_amortize_time_cost || old_garbage_is_cheaper || is_major_urgent(stats);
+}
+
+static double calculate_young_to_old_worker_ratio(const ZDirectorStats& stats) {
+  const double young_gc_time = gc_time(stats._young_stats);
+  const double old_gc_time = gc_time(stats._old_stats);
+  const size_t reclaimed_per_young_gc = stats._young_stats._stat_heap._reclaimed_avg;
+  const size_t reclaimed_per_old_gc = stats._old_stats._stat_heap._reclaimed_avg;
+  const double current_young_bytes_freed_per_gc_time = double(reclaimed_per_young_gc) / double(young_gc_time);
+  const double current_old_bytes_freed_per_gc_time = double(reclaimed_per_old_gc) / double(old_gc_time);
+  const double old_vs_young_efficiency_ratio = current_old_bytes_freed_per_gc_time / current_young_bytes_freed_per_gc_time;
+
+  return old_vs_young_efficiency_ratio;
+}
+
+static bool rule_major_proactive(const ZDirectorStats& stats) {
+  if (ZCollectionIntervalOnly) {
+    // Rule disabled
+    return false;
+  }
+
+  if (!ZProactive) {
+    // Rule disabled
+    return false;
+  }
+
+  if (!stats._old_stats._cycle._is_warm) {
+    // Rule disabled
+    return false;
   }
 
   // Perform GC if the impact of doing so, in terms of application throughput
@@ -333,74 +560,359 @@ static ZDriverRequest rule_proactive() {
   // 10% of the max capacity since the previous GC, or more than 5 minutes has
   // passed since the previous GC. This helps avoid superfluous GCs when running
   // applications with very low allocation rate.
-  const size_t used_after_last_gc = ZStatHeap::used_at_relocate_end();
-  const size_t used_increase_threshold = ZHeap::heap()->soft_max_capacity() * 0.10; // 10%
+  const size_t used_after_last_gc = stats._old_stats._stat_heap._used_at_relocate_end;
+  const size_t used_increase_threshold = stats._heap._soft_max_heap_size * 0.10; // 10%
   const size_t used_threshold = used_after_last_gc + used_increase_threshold;
-  const size_t used = ZHeap::heap()->used();
-  const double time_since_last_gc = ZStatCycle::time_since_last();
+  const size_t used = stats._heap._used;
+  const double time_since_last_gc = stats._old_stats._cycle._time_since_last;
   const double time_since_last_gc_threshold = 5 * 60; // 5 minutes
   if (used < used_threshold && time_since_last_gc < time_since_last_gc_threshold) {
     // Don't even consider doing a proactive GC
-    log_debug(gc, director)("Rule: Proactive, UsedUntilEnabled: " SIZE_FORMAT "MB, TimeUntilEnabled: %.3fs",
+    log_debug(gc, director)("Rule Major: Proactive, UsedUntilEnabled: " SIZE_FORMAT "MB, TimeUntilEnabled: %.3fs",
                             (used_threshold - used) / M,
                             time_since_last_gc_threshold - time_since_last_gc);
-    return GCCause::_no_gc;
+    return false;
   }
 
   const double assumed_throughput_drop_during_gc = 0.50; // 50%
   const double acceptable_throughput_drop = 0.01;        // 1%
-  const double serial_gc_time = ZStatCycle::serial_time().davg() + (ZStatCycle::serial_time().dsd() * one_in_1000);
-  const double parallelizable_gc_time = ZStatCycle::parallelizable_time().davg() + (ZStatCycle::parallelizable_time().dsd() * one_in_1000);
-  const double gc_duration = serial_gc_time + (parallelizable_gc_time / ConcGCThreads);
+  const double serial_old_gc_time = stats._old_stats._cycle._avg_serial_time + (stats._old_stats._cycle._sd_serial_time * one_in_1000);
+  const double parallelizable_old_gc_time = stats._old_stats._cycle._avg_parallelizable_time + (stats._old_stats._cycle._sd_parallelizable_time * one_in_1000);
+  const double serial_young_gc_time = stats._young_stats._cycle._avg_serial_time + (stats._young_stats._cycle._sd_serial_time * one_in_1000);
+  const double parallelizable_young_gc_time = stats._young_stats._cycle._avg_parallelizable_time + (stats._young_stats._cycle._sd_parallelizable_time * one_in_1000);
+  const double serial_gc_time = serial_old_gc_time + serial_young_gc_time;
+  const double parallelizable_gc_time = parallelizable_old_gc_time + parallelizable_young_gc_time;
+  const double gc_duration = serial_gc_time + parallelizable_gc_time;
   const double acceptable_gc_interval = gc_duration * ((assumed_throughput_drop_during_gc / acceptable_throughput_drop) - 1.0);
   const double time_until_gc = acceptable_gc_interval - time_since_last_gc;
 
-  log_debug(gc, director)("Rule: Proactive, AcceptableGCInterval: %.3fs, TimeSinceLastGC: %.3fs, TimeUntilGC: %.3fs",
+  log_debug(gc, director)("Rule Major: Proactive, AcceptableGCInterval: %.3fs, TimeSinceLastGC: %.3fs, TimeUntilGC: %.3fs",
                           acceptable_gc_interval, time_since_last_gc, time_until_gc);
 
-  if (time_until_gc > 0) {
+  return time_until_gc <= 0;
+}
+
+static GCCause::Cause make_minor_gc_decision(const ZDirectorStats& stats) {
+  if (ZDriver::minor()->is_busy()) {
     return GCCause::_no_gc;
   }
 
-  return GCCause::_z_proactive;
-}
+  if (ZDriver::major()->is_busy() && !stats._old_stats._resize._is_active) {
+    return GCCause::_no_gc;
+  }
 
-static ZDriverRequest make_gc_decision() {
-  // List of rules
-  using ZDirectorRule = ZDriverRequest (*)();
-  const ZDirectorRule rules[] = {
-    rule_allocation_stall,
-    rule_warmup,
-    rule_timer,
-    rule_allocation_rate,
-    rule_high_usage,
-    rule_proactive,
-  };
+  if (rule_minor_timer(stats)) {
+    return GCCause::_z_timer;
+  }
 
-  // Execute rules
-  for (size_t i = 0; i < ARRAY_SIZE(rules); i++) {
-    const ZDriverRequest request = rules[i]();
-    if (request.cause() != GCCause::_no_gc) {
-      return request;
-    }
+  if (rule_minor_allocation_rate(stats)) {
+    return GCCause::_z_allocation_rate;
+  }
+
+  if (rule_minor_high_usage(stats)) {
+    return GCCause::_z_high_usage;
   }
 
   return GCCause::_no_gc;
 }
 
-void ZDirector::run_service() {
+static GCCause::Cause make_major_gc_decision(const ZDirectorStats& stats) {
+  if (ZDriver::major()->is_busy()) {
+    return GCCause::_no_gc;
+  }
+
+  if (rule_major_timer(stats)) {
+    return GCCause::_z_timer;
+  }
+
+  if (rule_major_warmup(stats)) {
+    return GCCause::_z_warmup;
+  }
+
+  if (rule_major_proactive(stats)) {
+    return GCCause::_z_proactive;
+  }
+
+  return GCCause::_no_gc;
+}
+
+static ZWorkerResizeStats sample_worker_resize_stats(ZStatCycleStats& cycle_stats, ZStatWorkersStats& worker_stats, ZWorkers* workers) {
+  ZLocker<ZLock> locker(workers->resizing_lock());
+
+  if (!workers->is_active()) {
+    // If the workers are not active, it isn't safe to read stats
+    // from the stat_cycle, so return early.
+    return {
+      false, // _is_active
+      0.0,   // _serial_gc_time_passed
+      0.0,   // _parallel_gc_time_passed
+      0      // _nworkers_current
+    };
+  }
+
+  const double parallel_gc_duration_passed = worker_stats._accumulated_duration;
+  const double parallel_gc_time_passed = worker_stats._accumulated_time;
+  const double serial_gc_time_passed = cycle_stats._duration_since_start - parallel_gc_duration_passed;
+  const uint active_nworkers = workers->active_workers();
+
+  return {
+    true,                    // _is_active
+    serial_gc_time_passed,   // _serial_gc_time_passed
+    parallel_gc_time_passed, // _parallel_gc_time_passed
+    active_nworkers          // _nworkers_current
+  };
+}
+
+// Output information for select_worker_threads
+struct ZWorkerCounts {
+  uint _young_workers;
+  uint _old_workers;
+};
+
+enum class ZWorkerSelectionType {
+  start_major,
+  minor_during_old,
+  normal
+};
+
+static ZWorkerCounts select_worker_threads(const ZDirectorStats& stats, uint young_workers, ZWorkerSelectionType type) {
+  const uint active_young_workers = stats._young_stats._resize._nworkers_current;
+  const uint active_old_workers = stats._old_stats._resize._nworkers_current;
+
+  if (ZHeap::heap()->is_alloc_stalling()) {
+    // Boost GC threads when stalling
+    return {ZYoungGCThreads, ZOldGCThreads};
+  } else if (active_young_workers + active_old_workers > ConcGCThreads) {
+    // Threads are boosted, due to stalling recently; retain that boosting
+    return {active_young_workers, active_old_workers};
+  }
+
+  const double young_to_old_ratio = calculate_young_to_old_worker_ratio(stats);
+  uint old_workers = clamp(uint(young_workers * young_to_old_ratio), 1u, ZOldGCThreads);
+
+  if (type != ZWorkerSelectionType::normal && old_workers + young_workers > ConcGCThreads) {
+    // We need to somehow clamp the GC threads so the two generations don't exceed ConcGCThreads
+    const double old_ratio = (young_to_old_ratio / (1.0 + young_to_old_ratio));
+    const double young_ratio = 1.0 - old_ratio;
+    const uint young_workers_clamped = clamp(uint(ConcGCThreads * young_ratio), 1u, ZYoungGCThreads);
+    const uint old_workers_clamped = clamp(ConcGCThreads - young_workers_clamped, 1u, ZOldGCThreads);
+
+    if (type == ZWorkerSelectionType::start_major) {
+      // Adjust down the old workers so the next minor during major will be less sad
+      old_workers = old_workers_clamped;
+      // Since collecting the old generation depends on the initial young collection
+      // finishing, we don't want it to have fewer workers than the old generation.
+      young_workers = MAX2(old_workers, young_workers);
+    } else if (type == ZWorkerSelectionType::minor_during_old) {
+      // Adjust young and old workers for minor during old to fit within ConcGCThreads
+      young_workers = young_workers_clamped;
+      old_workers = old_workers_clamped;
+    }
+  }
+
+  return {young_workers, old_workers};
+}
+
+static void adjust_gc(const ZDirectorStats& stats) {
+  if (!UseDynamicNumberOfGCThreads) {
+    return;
+  }
+
+  const ZWorkerResizeStats young_resize_stats = stats._young_stats._resize;
+  const ZWorkerResizeStats old_resize_stats = stats._old_stats._resize;
+
+  if (!young_resize_stats._is_active) {
+    // Young generation collection is not running. We only resize the number
+    // of threads when the young generation is running. The number of threads
+    // for the old generation is modelled as a ratio of the number of threads
+    // needed in the young generation. If we don't need to GC the young generation
+    // at all, then we don't have anything to scale with, and the allocation
+    // pressure on the GC can't be that high. If it is, a minor collection will
+    // start, and inform us how to scale the old threads.
+    return;
+  }
+
+  const ZDriverRequest request = rule_semi_hard_minor_allocation_rate_dynamic(stats,
+                                                                              young_resize_stats._serial_gc_time_passed,
+                                                                              young_resize_stats._parallel_gc_time_passed);
+  if (request.cause() == GCCause::_no_gc) {
+    // No urgency
+    return;
+  }
+
+  uint desired_young_workers = MAX2(request.young_nworkers(), young_resize_stats._nworkers_current);
+
+  if (desired_young_workers > young_resize_stats._nworkers_current) {
+    // We need to increase workers
+    const uint needed_young_increase = desired_young_workers - young_resize_stats._nworkers_current;
+    // We want to increase by more than the minimum amount to ensure that
+    // there are enough margins, but also to avoid too frequent resizing.
+    const uint desired_young_increase = needed_young_increase * 2;
+    desired_young_workers = MIN2(young_resize_stats._nworkers_current + desired_young_increase, ZYoungGCThreads);
+  }
+
+  const uint young_current_workers = young_resize_stats._nworkers_current;
+  const uint old_current_workers = old_resize_stats._nworkers_current;
+
+  const bool minor_during_old = old_resize_stats._is_active;
+  ZWorkerSelectionType type = minor_during_old ? ZWorkerSelectionType::minor_during_old
+                                               : ZWorkerSelectionType::normal;
+
+  const ZWorkerCounts selection = select_worker_threads(stats, desired_young_workers, type);
+
+  if (old_resize_stats._is_active && old_current_workers != selection._old_workers) {
+    ZGeneration::old()->workers()->request_resize_workers(selection._old_workers);
+  }
+  if (young_current_workers != selection._young_workers) {
+    ZGeneration::young()->workers()->request_resize_workers(selection._young_workers);
+  }
+}
+
+static ZWorkerCounts initial_workers(const ZDirectorStats& stats, ZWorkerSelectionType type) {
+  if (!UseDynamicNumberOfGCThreads) {
+    return {ZYoungGCThreads, ZOldGCThreads};
+  }
+
+  const ZDriverRequest soft_request = rule_soft_minor_allocation_rate_dynamic(stats, 0.0 /* serial_gc_time_passed */, 0.0 /* parallel_gc_time_passed */);
+  const ZDriverRequest hard_request = rule_hard_minor_allocation_rate_dynamic(stats, 0.0 /* serial_gc_time_passed */, 0.0 /* parallel_gc_time_passed */);
+  const uint young_workers = MAX3(1u, soft_request.young_nworkers(), hard_request.young_nworkers());
+
+  return select_worker_threads(stats, young_workers, type);
+}
+
+static void start_major_gc(const ZDirectorStats& stats, GCCause::Cause cause) {
+  const ZWorkerCounts selection = initial_workers(stats, ZWorkerSelectionType::start_major);
+  const ZDriverRequest request(cause, selection._young_workers, selection._old_workers);
+  ZDriver::major()->collect(request);
+}
+
+static void start_minor_gc(const ZDirectorStats& stats, GCCause::Cause cause) {
+  const ZWorkerSelectionType type = ZDriver::major()->is_busy()
+      ? ZWorkerSelectionType::minor_during_old
+      : ZWorkerSelectionType::normal;
+  const ZWorkerCounts selection = initial_workers(stats, type);
+
+  if (UseDynamicNumberOfGCThreads && ZDriver::major()->is_busy()) {
+    const ZWorkerResizeStats old_resize_stats = stats._old_stats._resize;
+    const uint old_current_workers = old_resize_stats._nworkers_current;
+
+    if (old_current_workers != selection._old_workers) {
+      ZGeneration::old()->workers()->request_resize_workers(selection._old_workers);
+    }
+  }
+
+  const ZDriverRequest request(cause, selection._young_workers, 0);
+  ZDriver::minor()->collect(request);
+}
+
+static bool start_gc(const ZDirectorStats& stats) {
+  // Try start major collections first as they include a minor collection
+  const GCCause::Cause major_cause = make_major_gc_decision(stats);
+  if (major_cause != GCCause::_no_gc) {
+    start_major_gc(stats, major_cause);
+    return true;
+  }
+
+  const GCCause::Cause minor_cause = make_minor_gc_decision(stats);
+  if (minor_cause != GCCause::_no_gc) {
+    if (!ZDriver::major()->is_busy() && rule_major_allocation_rate(stats)) {
+      // Merge minor GC into major GC
+      start_major_gc(stats, GCCause::_z_allocation_rate);
+    } else {
+      start_minor_gc(stats, minor_cause);
+    }
+
+    return true;
+  }
+
+  return false;
+}
+
+void ZDirector::evaluate_rules() {
+  ZLocker<ZConditionLock> locker(&_director->_monitor);
+  _director->_monitor.notify();
+}
+
+bool ZDirector::wait_for_tick() {
+  const uint64_t interval_ms = MILLIUNITS / decision_hz;
+
+  ZLocker<ZConditionLock> locker(&_monitor);
+
+  if (_stopped) {
+    // Stopped
+    return false;
+  }
+
+  // Wait
+  _monitor.wait(interval_ms);
+  return true;
+}
+
+static ZDirectorHeapStats sample_heap_stats() {
+  const ZHeap* const heap = ZHeap::heap();
+  const ZCollectedHeap* const collected_heap = ZCollectedHeap::heap();
+  return {
+    heap->soft_max_capacity(),
+    heap->used(),
+    collected_heap->total_collections()
+  };
+}
+
+// This function samples all the stat values used by the heuristics to compute what to do.
+// This is where synchronization code goes to ensure that the values we read are valid.
+static ZDirectorStats sample_stats() {
+  ZGenerationYoung* young = ZGeneration::young();
+  ZGenerationOld* old = ZGeneration::old();
+  const ZStatMutatorAllocRateStats mutator_alloc_rate = ZStatMutatorAllocRate::stats();
+  const ZDirectorHeapStats heap = sample_heap_stats();
+
+  ZStatCycleStats young_cycle = young->stat_cycle()->stats();
+  ZStatCycleStats old_cycle = old->stat_cycle()->stats();
+
+  ZStatWorkersStats young_workers = young->stat_workers()->stats();
+  ZStatWorkersStats old_workers = old->stat_workers()->stats();
+
+  ZWorkerResizeStats young_resize = sample_worker_resize_stats(young_cycle, young_workers, young->workers());
+  ZWorkerResizeStats old_resize = sample_worker_resize_stats(old_cycle, old_workers, old->workers());
+
+  ZStatHeapStats young_stat_heap = young->stat_heap()->stats();
+  ZStatHeapStats old_stat_heap = old->stat_heap()->stats();
+
+  ZDirectorGenerationGeneralStats young_generation = { ZHeap::heap()->used_young(), 0 };
+  ZDirectorGenerationGeneralStats old_generation = { ZHeap::heap()->used_old(), old->total_collections_at_start() };
+
+  return {
+    mutator_alloc_rate,
+    heap,
+    {
+      young_cycle,
+      young_workers,
+      young_resize,
+      young_stat_heap,
+      young_generation
+    },
+    {
+      old_cycle,
+      old_workers,
+      old_resize,
+      old_stat_heap,
+      old_generation
+    }
+  };
+}
+
+void ZDirector::run_thread() {
   // Main loop
-  while (_metronome.wait_for_tick()) {
-    sample_allocation_rate();
-    if (!_driver->is_busy()) {
-      const ZDriverRequest request = make_gc_decision();
-      if (request.cause() != GCCause::_no_gc) {
-        _driver->collect(request);
-      }
+  while (wait_for_tick()) {
+    ZDirectorStats stats = sample_stats();
+    if (!start_gc(stats)) {
+      adjust_gc(stats);
     }
   }
 }
 
-void ZDirector::stop_service() {
-  _metronome.stop();
+void ZDirector::terminate() {
+  ZLocker<ZConditionLock> locker(&_monitor);
+  _stopped = true;
+  _monitor.notify();
 }
diff --git a/src/hotspot/share/gc/z/zDirector.hpp b/src/hotspot/share/gc/z/zDirector.hpp
index a46c4ba7702..73c556a2fbd 100644
--- a/src/hotspot/share/gc/z/zDirector.hpp
+++ b/src/hotspot/share/gc/z/zDirector.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,22 +24,27 @@
 #ifndef SHARE_GC_Z_ZDIRECTOR_HPP
 #define SHARE_GC_Z_ZDIRECTOR_HPP
 
-#include "gc/shared/concurrentGCThread.hpp"
-#include "gc/z/zMetronome.hpp"
+#include "gc/z/zLock.hpp"
+#include "gc/z/zThread.hpp"
 
-class ZDriver;
-
-class ZDirector : public ConcurrentGCThread {
+class ZDirector : public ZThread {
 private:
-  ZDriver* const _driver;
-  ZMetronome     _metronome;
+  static const uint64_t decision_hz = 100;
+  static ZDirector* _director;
+
+  ZConditionLock _monitor;
+  bool           _stopped;
+
+  bool wait_for_tick();
 
 protected:
-  virtual void run_service();
-  virtual void stop_service();
+  virtual void run_thread();
+  virtual void terminate();
 
 public:
-  ZDirector(ZDriver* driver);
+  ZDirector();
+
+  static void evaluate_rules();
 };
 
 #endif // SHARE_GC_Z_ZDIRECTOR_HPP
diff --git a/src/hotspot/share/gc/z/zDriver.cpp b/src/hotspot/share/gc/z/zDriver.cpp
index 4388febe280..f8ae47399e5 100644
--- a/src/hotspot/share/gc/z/zDriver.cpp
+++ b/src/hotspot/share/gc/z/zDriver.cpp
@@ -22,333 +22,246 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/shared/gcCause.hpp"
 #include "gc/shared/gcId.hpp"
-#include "gc/shared/gcLocker.hpp"
-#include "gc/shared/gcVMOperations.hpp"
-#include "gc/shared/isGCActiveMark.hpp"
 #include "gc/z/zAbort.inline.hpp"
 #include "gc/z/zBreakpoint.hpp"
 #include "gc/z/zCollectedHeap.hpp"
+#include "gc/z/zDirector.hpp"
 #include "gc/z/zDriver.hpp"
+#include "gc/z/zGCIdPrinter.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zHeap.inline.hpp"
-#include "gc/z/zMessagePort.inline.hpp"
+#include "gc/z/zLock.inline.hpp"
 #include "gc/z/zServiceability.hpp"
 #include "gc/z/zStat.hpp"
-#include "gc/z/zVerify.hpp"
-#include "logging/log.hpp"
-#include "memory/universe.hpp"
-#include "runtime/threads.hpp"
-#include "runtime/vmOperations.hpp"
-#include "runtime/vmThread.hpp"
 
-static const ZStatPhaseCycle      ZPhaseCycle("Garbage Collection Cycle");
-static const ZStatPhasePause      ZPhasePauseMarkStart("Pause Mark Start");
-static const ZStatPhaseConcurrent ZPhaseConcurrentMark("Concurrent Mark");
-static const ZStatPhaseConcurrent ZPhaseConcurrentMarkContinue("Concurrent Mark Continue");
-static const ZStatPhaseConcurrent ZPhaseConcurrentMarkFree("Concurrent Mark Free");
-static const ZStatPhasePause      ZPhasePauseMarkEnd("Pause Mark End");
-static const ZStatPhaseConcurrent ZPhaseConcurrentProcessNonStrongReferences("Concurrent Process Non-Strong References");
-static const ZStatPhaseConcurrent ZPhaseConcurrentResetRelocationSet("Concurrent Reset Relocation Set");
-static const ZStatPhaseConcurrent ZPhaseConcurrentSelectRelocationSet("Concurrent Select Relocation Set");
-static const ZStatPhasePause      ZPhasePauseRelocateStart("Pause Relocate Start");
-static const ZStatPhaseConcurrent ZPhaseConcurrentRelocated("Concurrent Relocate");
-static const ZStatCriticalPhase   ZCriticalPhaseGCLockerStall("GC Locker Stall", false /* verbose */);
-static const ZStatSampler         ZSamplerJavaThreads("System", "Java Threads", ZStatUnitThreads);
+static const ZStatPhaseCollection ZPhaseCollectionMinor("Minor Collection", true /* minor */);
+static const ZStatPhaseCollection ZPhaseCollectionMajor("Major Collection", false /* minor */);
 
-ZDriverRequest::ZDriverRequest() :
-    ZDriverRequest(GCCause::_no_gc) {}
-
-ZDriverRequest::ZDriverRequest(GCCause::Cause cause) :
-    ZDriverRequest(cause, ConcGCThreads) {}
-
-ZDriverRequest::ZDriverRequest(GCCause::Cause cause, uint nworkers) :
-    _cause(cause),
-    _nworkers(nworkers) {}
-
-bool ZDriverRequest::operator==(const ZDriverRequest& other) const {
-  return _cause == other._cause;
-}
-
-GCCause::Cause ZDriverRequest::cause() const {
-  return _cause;
-}
-
-uint ZDriverRequest::nworkers() const {
-  return _nworkers;
-}
-
-class VM_ZOperation : public VM_Operation {
+template <typename DriverT>
+class ZGCCauseSetter : public GCCauseSetter {
 private:
-  const uint _gc_id;
-  bool       _gc_locked;
-  bool       _success;
+  DriverT* _driver;
 
 public:
-  VM_ZOperation() :
-      _gc_id(GCId::current()),
-      _gc_locked(false),
-      _success(false) {}
-
-  virtual bool needs_inactive_gc_locker() const {
-    // An inactive GC locker is needed in operations where we change the bad
-    // mask or move objects. Changing the bad mask will invalidate all oops,
-    // which makes it conceptually the same thing as moving all objects.
-    return false;
+  ZGCCauseSetter(DriverT* driver, GCCause::Cause cause) :
+      GCCauseSetter(ZCollectedHeap::heap(), cause),
+      _driver(driver) {
+    _driver->set_gc_cause(cause);
   }
 
-  virtual bool skip_thread_oop_barriers() const {
-    return true;
-  }
-
-  virtual bool do_operation() = 0;
-
-  virtual bool doit_prologue() {
-    Heap_lock->lock();
-    return true;
-  }
-
-  virtual void doit() {
-    // Abort if GC locker state is incompatible
-    if (needs_inactive_gc_locker() && GCLocker::check_active_before_gc()) {
-      _gc_locked = true;
-      return;
-    }
-
-    // Setup GC id and active marker
-    GCIdMark gc_id_mark(_gc_id);
-    IsGCActiveMark gc_active_mark;
-
-    // Verify before operation
-    ZVerify::before_zoperation();
-
-    // Execute operation
-    _success = do_operation();
-
-    // Update statistics
-    ZStatSample(ZSamplerJavaThreads, Threads::number_of_threads());
-  }
-
-  virtual void doit_epilogue() {
-    Heap_lock->unlock();
-  }
-
-  bool gc_locked() const {
-    return _gc_locked;
-  }
-
-  bool success() const {
-    return _success;
+  ~ZGCCauseSetter() {
+    _driver->set_gc_cause(GCCause::_no_gc);
   }
 };
 
-class VM_ZMarkStart : public VM_ZOperation {
-public:
-  virtual VMOp_Type type() const {
-    return VMOp_ZMarkStart;
-  }
+ZLock*        ZDriver::_lock;
+ZDriverMinor* ZDriver::_minor;
+ZDriverMajor* ZDriver::_major;
 
-  virtual bool needs_inactive_gc_locker() const {
-    return true;
-  }
+void ZDriver::initialize() {
+  _lock = new ZLock();
+}
 
-  virtual bool do_operation() {
-    ZStatTimer timer(ZPhasePauseMarkStart);
-    ZServiceabilityPauseTracer tracer;
+void ZDriver::lock() {
+  _lock->lock();
+}
 
-    ZCollectedHeap::heap()->increment_total_collections(true /* full */);
+void ZDriver::unlock() {
+  _lock->unlock();
+}
 
-    ZHeap::heap()->mark_start();
-    return true;
-  }
-};
+void ZDriver::set_minor(ZDriverMinor* minor) {
+  _minor = minor;
+}
 
-class VM_ZMarkEnd : public VM_ZOperation {
-public:
-  virtual VMOp_Type type() const {
-    return VMOp_ZMarkEnd;
-  }
+void ZDriver::set_major(ZDriverMajor* major) {
+  _major = major;
+}
 
-  virtual bool do_operation() {
-    ZStatTimer timer(ZPhasePauseMarkEnd);
-    ZServiceabilityPauseTracer tracer;
-    return ZHeap::heap()->mark_end();
-  }
-};
+ZDriverMinor* ZDriver::minor() {
+  return _minor;
+}
 
-class VM_ZRelocateStart : public VM_ZOperation {
-public:
-  virtual VMOp_Type type() const {
-    return VMOp_ZRelocateStart;
-  }
+ZDriverMajor* ZDriver::major() {
+  return _major;
+}
 
-  virtual bool needs_inactive_gc_locker() const {
-    return true;
-  }
+ZDriverLocker::ZDriverLocker() {
+  ZDriver::lock();
+}
 
-  virtual bool do_operation() {
-    ZStatTimer timer(ZPhasePauseRelocateStart);
-    ZServiceabilityPauseTracer tracer;
-    ZHeap::heap()->relocate_start();
-    return true;
-  }
-};
+ZDriverLocker::~ZDriverLocker() {
+  ZDriver::unlock();
+}
 
-class VM_ZVerify : public VM_Operation {
-public:
-  virtual VMOp_Type type() const {
-    return VMOp_ZVerify;
-  }
+ZDriverUnlocker::ZDriverUnlocker() {
+  ZDriver::unlock();
+}
 
-  virtual bool skip_thread_oop_barriers() const {
-    return true;
-  }
-
-  virtual void doit() {
-    ZVerify::after_weak_processing();
-  }
-};
+ZDriverUnlocker::~ZDriverUnlocker() {
+  ZDriver::lock();
+}
 
 ZDriver::ZDriver() :
-    _gc_cycle_port(),
-    _gc_locker_port() {
-  set_name("ZDriver");
+    _gc_cause(GCCause::_no_gc) {
+}
+
+void ZDriver::set_gc_cause(GCCause::Cause cause) {
+  _gc_cause = cause;
+}
+
+GCCause::Cause ZDriver::gc_cause() {
+  return _gc_cause;
+}
+
+ZDriverMinor::ZDriverMinor() :
+    ZDriver(),
+    _port(),
+    _gc_timer(),
+    _jfr_tracer(),
+    _used_at_start() {
+  ZDriver::set_minor(this);
+  set_name("ZDriverMinor");
   create_and_start();
 }
 
-bool ZDriver::is_busy() const {
-  return _gc_cycle_port.is_busy();
+bool ZDriverMinor::is_busy() const {
+  return _port.is_busy();
 }
 
-void ZDriver::collect(const ZDriverRequest& request) {
+void ZDriverMinor::collect(const ZDriverRequest& request) {
   switch (request.cause()) {
   case GCCause::_wb_young_gc:
-  case GCCause::_wb_full_gc:
-  case GCCause::_dcmd_gc_run:
-  case GCCause::_java_lang_system_gc:
-  case GCCause::_full_gc_alot:
-  case GCCause::_scavenge_alot:
-  case GCCause::_jvmti_force_gc:
-  case GCCause::_metadata_GC_clear_soft_refs:
-  case GCCause::_codecache_GC_aggressive:
     // Start synchronous GC
-    _gc_cycle_port.send_sync(request);
+    _port.send_sync(request);
     break;
 
+  case GCCause::_scavenge_alot:
   case GCCause::_z_timer:
-  case GCCause::_z_warmup:
   case GCCause::_z_allocation_rate:
   case GCCause::_z_allocation_stall:
-  case GCCause::_z_proactive:
   case GCCause::_z_high_usage:
-  case GCCause::_codecache_GC_threshold:
-  case GCCause::_metadata_GC_threshold:
     // Start asynchronous GC
-    _gc_cycle_port.send_async(request);
-    break;
-
-  case GCCause::_gc_locker:
-    // Restart VM operation previously blocked by the GC locker
-    _gc_locker_port.signal();
-    break;
-
-  case GCCause::_wb_breakpoint:
-    ZBreakpoint::start_gc();
-    _gc_cycle_port.send_async(request);
+    _port.send_async(request);
     break;
 
   default:
-    // Other causes not supported
     fatal("Unsupported GC cause (%s)", GCCause::to_string(request.cause()));
     break;
   }
+};
+
+GCTracer* ZDriverMinor::jfr_tracer() {
+  return &_jfr_tracer;
 }
 
-template <typename T>
-bool ZDriver::pause() {
+void ZDriverMinor::set_used_at_start(size_t used) {
+  _used_at_start = used;
+}
+
+size_t ZDriverMinor::used_at_start() const {
+  return _used_at_start;
+}
+
+class ZDriverScopeMinor : public StackObj {
+private:
+  GCIdMark                     _gc_id;
+  GCCause::Cause               _gc_cause;
+  ZGCCauseSetter<ZDriverMinor> _gc_cause_setter;
+  ZStatTimer                   _stat_timer;
+  ZServiceabilityCycleTracer   _tracer;
+
+public:
+  ZDriverScopeMinor(const ZDriverRequest& request, ConcurrentGCTimer* gc_timer) :
+      _gc_id(),
+      _gc_cause(request.cause()),
+      _gc_cause_setter(ZDriver::minor(), _gc_cause),
+      _stat_timer(ZPhaseCollectionMinor, gc_timer),
+      _tracer(true /* minor */) {
+    // Select number of worker threads to use
+    ZGeneration::young()->set_active_workers(request.young_nworkers());
+  }
+};
+
+void ZDriverMinor::gc(const ZDriverRequest& request) {
+  ZDriverScopeMinor scope(request, &_gc_timer);
+  ZGCIdMinor minor_id(gc_id());
+  ZGeneration::young()->collect(ZYoungType::minor, &_gc_timer);
+}
+
+static void handle_alloc_stalling_for_young() {
+  ZHeap::heap()->handle_alloc_stalling_for_young();
+}
+
+void ZDriverMinor::handle_alloc_stalls() const {
+  handle_alloc_stalling_for_young();
+}
+
+void ZDriverMinor::run_thread() {
+  // Main loop
   for (;;) {
-    T op;
-    VMThread::execute(&op);
-    if (op.gc_locked()) {
-      // Wait for GC to become unlocked and restart the VM operation
-      ZStatTimer timer(ZCriticalPhaseGCLockerStall);
-      _gc_locker_port.wait();
-      continue;
-    }
+    // Wait for GC request
+    const ZDriverRequest request = _port.receive();
 
-    // Notify VM operation completed
-    _gc_locker_port.ack();
+    ZDriverLocker locker;
 
-    return op.success();
+    abortpoint();
+
+    // Run GC
+    gc(request);
+
+    abortpoint();
+
+    // Notify GC completed
+    _port.ack();
+
+    // Handle allocation stalls
+    handle_alloc_stalls();
+
+    // Good point to consider back-to-back GC
+    ZDirector::evaluate_rules();
   }
 }
 
-void ZDriver::pause_mark_start() {
-  pause<VM_ZMarkStart>();
+void ZDriverMinor::terminate() {
+  const ZDriverRequest request(GCCause::_no_gc, 0, 0);
+  _port.send_async(request);
 }
 
-void ZDriver::concurrent_mark() {
-  ZStatTimer timer(ZPhaseConcurrentMark);
-  ZBreakpoint::at_after_marking_started();
-  ZHeap::heap()->mark(true /* initial */);
-  ZBreakpoint::at_before_marking_completed();
-}
-
-bool ZDriver::pause_mark_end() {
-  return pause<VM_ZMarkEnd>();
-}
-
-void ZDriver::concurrent_mark_continue() {
-  ZStatTimer timer(ZPhaseConcurrentMarkContinue);
-  ZHeap::heap()->mark(false /* initial */);
-}
-
-void ZDriver::concurrent_mark_free() {
-  ZStatTimer timer(ZPhaseConcurrentMarkFree);
-  ZHeap::heap()->mark_free();
-}
-
-void ZDriver::concurrent_process_non_strong_references() {
-  ZStatTimer timer(ZPhaseConcurrentProcessNonStrongReferences);
-  ZBreakpoint::at_after_reference_processing_started();
-  ZHeap::heap()->process_non_strong_references();
-}
-
-void ZDriver::concurrent_reset_relocation_set() {
-  ZStatTimer timer(ZPhaseConcurrentResetRelocationSet);
-  ZHeap::heap()->reset_relocation_set();
-}
-
-void ZDriver::pause_verify() {
-  if (ZVerifyRoots || ZVerifyObjects) {
-    VM_ZVerify op;
-    VMThread::execute(&op);
-  }
-}
-
-void ZDriver::concurrent_select_relocation_set() {
-  ZStatTimer timer(ZPhaseConcurrentSelectRelocationSet);
-  ZHeap::heap()->select_relocation_set();
-}
-
-void ZDriver::pause_relocate_start() {
-  pause<VM_ZRelocateStart>();
-}
-
-void ZDriver::concurrent_relocate() {
-  ZStatTimer timer(ZPhaseConcurrentRelocated);
-  ZHeap::heap()->relocate();
-}
-
-void ZDriver::check_out_of_memory() {
-  ZHeap::heap()->check_out_of_memory();
-}
-
-static bool should_clear_soft_references(const ZDriverRequest& request) {
+static bool should_clear_soft_references(GCCause::Cause cause) {
   // Clear soft references if implied by the GC cause
-  if (request.cause() == GCCause::_wb_full_gc ||
-      request.cause() == GCCause::_metadata_GC_clear_soft_refs ||
-      request.cause() == GCCause::_z_allocation_stall) {
-    // Clear
+  switch (cause) {
+  case GCCause::_wb_full_gc:
+  case GCCause::_metadata_GC_clear_soft_refs:
+  case GCCause::_z_allocation_stall:
+    return true;
+
+  case GCCause::_heap_dump:
+  case GCCause::_heap_inspection:
+  case GCCause::_wb_breakpoint:
+  case GCCause::_dcmd_gc_run:
+  case GCCause::_java_lang_system_gc:
+  case GCCause::_full_gc_alot:
+  case GCCause::_jvmti_force_gc:
+  case GCCause::_z_timer:
+  case GCCause::_z_warmup:
+  case GCCause::_z_allocation_rate:
+  case GCCause::_z_proactive:
+  case GCCause::_metadata_GC_threshold:
+  case GCCause::_codecache_GC_threshold:
+  case GCCause::_codecache_GC_aggressive:
+    break;
+
+  default:
+    fatal("Unsupported GC cause (%s)", GCCause::to_string(cause));
+    break;
+  }
+
+  // Clear soft references if threads are stalled waiting for an old collection
+  if (ZHeap::heap()->is_alloc_stalling_for_old()) {
     return true;
   }
 
@@ -356,156 +269,215 @@ static bool should_clear_soft_references(const ZDriverRequest& request) {
   return false;
 }
 
-static uint select_active_worker_threads_dynamic(const ZDriverRequest& request) {
-  // Use requested number of worker threads
-  return request.nworkers();
-}
+static bool should_preclean_young(GCCause::Cause cause) {
+  // Preclean young if implied by the GC cause
+  switch (cause) {
+  case GCCause::_heap_dump:
+  case GCCause::_heap_inspection:
+  case GCCause::_wb_full_gc:
+  case GCCause::_wb_breakpoint:
+  case GCCause::_dcmd_gc_run:
+  case GCCause::_java_lang_system_gc:
+  case GCCause::_full_gc_alot:
+  case GCCause::_jvmti_force_gc:
+  case GCCause::_metadata_GC_clear_soft_refs:
+  case GCCause::_z_allocation_stall:
+    return true;
 
-static uint select_active_worker_threads_static(const ZDriverRequest& request) {
-  const GCCause::Cause cause = request.cause();
-  const uint nworkers = request.nworkers();
+  case GCCause::_z_timer:
+  case GCCause::_z_warmup:
+  case GCCause::_z_allocation_rate:
+  case GCCause::_z_proactive:
+  case GCCause::_metadata_GC_threshold:
+  case GCCause::_codecache_GC_threshold:
+  case GCCause::_codecache_GC_aggressive:
+    break;
 
-  // Boost number of worker threads if implied by the GC cause
-  if (cause == GCCause::_wb_full_gc ||
-      cause == GCCause::_java_lang_system_gc ||
-      cause == GCCause::_metadata_GC_clear_soft_refs ||
-      cause == GCCause::_z_allocation_stall) {
-    // Boost
-    const uint boosted_nworkers = MAX2(nworkers, ParallelGCThreads);
-    return boosted_nworkers;
+  default:
+    fatal("Unsupported GC cause (%s)", GCCause::to_string(cause));
+    break;
   }
 
-  // Use requested number of worker threads
-  return nworkers;
+  // Preclean young if threads are stalled waiting for an old collection
+  if (ZHeap::heap()->is_alloc_stalling_for_old()) {
+    return true;
+  }
+
+  // Preclean young if implied by configuration
+  return ScavengeBeforeFullGC;
 }
 
-static uint select_active_worker_threads(const ZDriverRequest& request) {
-  if (UseDynamicNumberOfGCThreads) {
-    return select_active_worker_threads_dynamic(request);
-  } else {
-    return select_active_worker_threads_static(request);
+ZDriverMajor::ZDriverMajor() :
+    ZDriver(),
+    _port(),
+    _gc_timer(),
+    _jfr_tracer(),
+    _used_at_start() {
+  ZDriver::set_major(this);
+  set_name("ZDriverMajor");
+  create_and_start();
+}
+
+bool ZDriverMajor::is_busy() const {
+  return _port.is_busy();
+}
+
+void ZDriverMajor::collect(const ZDriverRequest& request) {
+  switch (request.cause()) {
+  case GCCause::_heap_dump:
+  case GCCause::_heap_inspection:
+  case GCCause::_wb_full_gc:
+  case GCCause::_dcmd_gc_run:
+  case GCCause::_java_lang_system_gc:
+  case GCCause::_full_gc_alot:
+  case GCCause::_jvmti_force_gc:
+  case GCCause::_metadata_GC_clear_soft_refs:
+  case GCCause::_codecache_GC_aggressive:
+    // Start synchronous GC
+    _port.send_sync(request);
+    break;
+
+  case GCCause::_z_timer:
+  case GCCause::_z_warmup:
+  case GCCause::_z_allocation_rate:
+  case GCCause::_z_allocation_stall:
+  case GCCause::_z_proactive:
+  case GCCause::_codecache_GC_threshold:
+  case GCCause::_metadata_GC_threshold:
+    // Start asynchronous GC
+    _port.send_async(request);
+    break;
+
+  case GCCause::_wb_breakpoint:
+    ZBreakpoint::start_gc();
+    _port.send_async(request);
+    break;
+
+  default:
+    fatal("Unsupported GC cause (%s)", GCCause::to_string(request.cause()));
+    break;
   }
 }
 
-class ZDriverGCScope : public StackObj {
+GCTracer* ZDriverMajor::jfr_tracer() {
+  return &_jfr_tracer;
+}
+
+void ZDriverMajor::set_used_at_start(size_t used) {
+  _used_at_start = used;
+}
+
+size_t ZDriverMajor::used_at_start() const {
+  return _used_at_start;
+}
+
+class ZDriverScopeMajor : public StackObj {
 private:
-  GCIdMark                   _gc_id;
-  GCCause::Cause             _gc_cause;
-  GCCauseSetter              _gc_cause_setter;
-  ZStatTimer                 _timer;
-  ZServiceabilityCycleTracer _tracer;
+  GCIdMark                     _gc_id;
+  GCCause::Cause               _gc_cause;
+  ZGCCauseSetter<ZDriverMajor> _gc_cause_setter;
+  ZStatTimer                   _stat_timer;
+  ZServiceabilityCycleTracer   _tracer;
 
 public:
-  ZDriverGCScope(const ZDriverRequest& request) :
+  ZDriverScopeMajor(const ZDriverRequest& request, ConcurrentGCTimer* gc_timer) :
       _gc_id(),
       _gc_cause(request.cause()),
-      _gc_cause_setter(ZCollectedHeap::heap(), _gc_cause),
-      _timer(ZPhaseCycle),
-      _tracer() {
-    // Update statistics
-    ZStatCycle::at_start();
-
+      _gc_cause_setter(ZDriver::major(), _gc_cause),
+      _stat_timer(ZPhaseCollectionMajor, gc_timer),
+      _tracer(false /* minor */) {
     // Set up soft reference policy
-    const bool clear = should_clear_soft_references(request);
-    ZHeap::heap()->set_soft_reference_policy(clear);
+    const bool clear = should_clear_soft_references(request.cause());
+    ZGeneration::old()->set_soft_reference_policy(clear);
 
     // Select number of worker threads to use
-    const uint nworkers = select_active_worker_threads(request);
-    ZHeap::heap()->set_active_workers(nworkers);
+    ZGeneration::young()->set_active_workers(request.young_nworkers());
+    ZGeneration::old()->set_active_workers(request.old_nworkers());
   }
 
-  ~ZDriverGCScope() {
-    // Update statistics
-    ZStatCycle::at_end(_gc_cause, ZHeap::heap()->active_workers());
-
+  ~ZDriverScopeMajor() {
     // Update data used by soft reference policy
-    Universe::heap()->update_capacity_and_used_at_gc();
+    ZCollectedHeap::heap()->update_capacity_and_used_at_gc();
 
     // Signal that we have completed a visit to all live objects
-    Universe::heap()->record_whole_heap_examined_timestamp();
+    ZCollectedHeap::heap()->record_whole_heap_examined_timestamp();
   }
 };
 
-// Macro to execute a termination check after a concurrent phase. Note
-// that it's important that the termination check comes after the call
-// to the function f, since we can't abort between pause_relocate_start()
-// and concurrent_relocate(). We need to let concurrent_relocate() call
-// abort_page() on the remaining entries in the relocation set.
-#define concurrent(f)                 \
-  do {                                \
-    concurrent_##f();                 \
-    if (should_terminate()) {         \
-      return;                         \
-    }                                 \
-  } while (false)
+void ZDriverMajor::collect_young(const ZDriverRequest& request) {
+  ZGCIdMajor major_id(gc_id(), 'Y');
+  if (should_preclean_young(request.cause())) {
+    // Collect young generation and promote everything to old generation
+    ZGeneration::young()->collect(ZYoungType::major_full_preclean, &_gc_timer);
 
-void ZDriver::gc(const ZDriverRequest& request) {
-  ZDriverGCScope scope(request);
+    abortpoint();
 
-  // Phase 1: Pause Mark Start
-  pause_mark_start();
-
-  // Phase 2: Concurrent Mark
-  concurrent(mark);
-
-  // Phase 3: Pause Mark End
-  while (!pause_mark_end()) {
-    // Phase 3.5: Concurrent Mark Continue
-    concurrent(mark_continue);
+    // Collect young generation and gather roots pointing into old generation
+    ZGeneration::young()->collect(ZYoungType::major_full_roots, &_gc_timer);
+  } else {
+    // Collect young generation and gather roots pointing into old generation
+    ZGeneration::young()->collect(ZYoungType::major_partial_roots, &_gc_timer);
   }
 
-  // Phase 4: Concurrent Mark Free
-  concurrent(mark_free);
+  abortpoint();
 
-  // Phase 5: Concurrent Process Non-Strong References
-  concurrent(process_non_strong_references);
-
-  // Phase 6: Concurrent Reset Relocation Set
-  concurrent(reset_relocation_set);
-
-  // Phase 7: Pause Verify
-  pause_verify();
-
-  // Phase 8: Concurrent Select Relocation Set
-  concurrent(select_relocation_set);
-
-  // Phase 9: Pause Relocate Start
-  pause_relocate_start();
-
-  // Phase 10: Concurrent Relocate
-  concurrent(relocate);
+  // Handle allocations waiting for a young collection
+  handle_alloc_stalling_for_young();
 }
 
-void ZDriver::run_service() {
+void ZDriverMajor::collect_old() {
+  ZGCIdMajor major_id(gc_id(), 'O');
+  ZGeneration::old()->collect(&_gc_timer);
+}
+
+void ZDriverMajor::gc(const ZDriverRequest& request) {
+  ZDriverScopeMajor scope(request, &_gc_timer);
+
+  // Collect the young generation
+  collect_young(request);
+
+  abortpoint();
+
+  // Collect the old generation
+  collect_old();
+}
+
+static void handle_alloc_stalling_for_old() {
+  ZHeap::heap()->handle_alloc_stalling_for_old();
+}
+
+void ZDriverMajor::handle_alloc_stalls() const {
+  handle_alloc_stalling_for_old();
+}
+
+void ZDriverMajor::run_thread() {
   // Main loop
-  while (!should_terminate()) {
+  for (;;) {
     // Wait for GC request
-    const ZDriverRequest request = _gc_cycle_port.receive();
-    if (request.cause() == GCCause::_no_gc) {
-      continue;
-    }
+    const ZDriverRequest request = _port.receive();
+
+    ZDriverLocker locker;
 
     ZBreakpoint::at_before_gc();
 
+    abortpoint();
+
     // Run GC
     gc(request);
 
-    if (should_terminate()) {
-      // Abort
-      break;
-    }
+    abortpoint();
 
     // Notify GC completed
-    _gc_cycle_port.ack();
+    _port.ack();
 
-    // Check for out of memory condition
-    check_out_of_memory();
+    // Handle allocation stalls
+    handle_alloc_stalls();
 
     ZBreakpoint::at_after_gc();
   }
 }
 
-void ZDriver::stop_service() {
-  ZAbort::abort();
-  _gc_cycle_port.send_async(GCCause::_no_gc);
+void ZDriverMajor::terminate() {
+  const ZDriverRequest request(GCCause::_no_gc, 0, 0);
+  _port.send_async(request);
 }
diff --git a/src/hotspot/share/gc/z/zDriver.hpp b/src/hotspot/share/gc/z/zDriver.hpp
index 08b1b80f2aa..5f1fe08a0b6 100644
--- a/src/hotspot/share/gc/z/zDriver.hpp
+++ b/src/hotspot/share/gc/z/zDriver.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,61 +24,123 @@
 #ifndef SHARE_GC_Z_ZDRIVER_HPP
 #define SHARE_GC_Z_ZDRIVER_HPP
 
-#include "gc/shared/concurrentGCThread.hpp"
-#include "gc/shared/gcCause.hpp"
-#include "gc/z/zMessagePort.hpp"
+#include "gc/shared/gcTimer.hpp"
+#include "gc/z/zDriverPort.hpp"
+#include "gc/z/zThread.hpp"
+#include "gc/z/zTracer.hpp"
+
+// glibc <sys/sysmacros.h> which may get brought in via <sys/types.h>
+// defines the macros minor and major. These keywords are central to
+// the GC algorithm. These macros are undefined here so the code may
+// use minor and major.
+#ifdef minor
+#undef minor
+#endif
+#ifdef major
+#undef major
+#endif
 
 class VM_ZOperation;
+class ZDriverMinor;
+class ZDriverMajor;
+class ZLock;
+
+class ZDriver : public ZThread {
+  friend class ZDriverLocker;
+  friend class ZDriverUnlocker;
 
-class ZDriverRequest {
 private:
-  GCCause::Cause _cause;
-  uint           _nworkers;
+  static ZLock*        _lock;
+  static ZDriverMinor* _minor;
+  static ZDriverMajor* _major;
+
+  GCCause::Cause _gc_cause;
+
+  static void lock();
+  static void unlock();
 
 public:
-  ZDriverRequest();
-  ZDriverRequest(GCCause::Cause cause);
-  ZDriverRequest(GCCause::Cause cause, uint nworkers);
+  static void initialize();
 
-  bool operator==(const ZDriverRequest& other) const;
+  static void set_minor(ZDriverMinor* minor);
+  static void set_major(ZDriverMajor* major);
 
-  GCCause::Cause cause() const;
-  uint nworkers() const;
+  static ZDriverMinor* minor();
+  static ZDriverMajor* major();
+
+  ZDriver();
+
+  void set_gc_cause(GCCause::Cause cause);
+  GCCause::Cause gc_cause();
 };
 
-class ZDriver : public ConcurrentGCThread {
+class ZDriverMinor : public ZDriver {
 private:
-  ZMessagePort<ZDriverRequest> _gc_cycle_port;
-  ZRendezvousPort              _gc_locker_port;
-
-  template <typename T> bool pause();
-
-  void pause_mark_start();
-  void concurrent_mark();
-  bool pause_mark_end();
-  void concurrent_mark_continue();
-  void concurrent_mark_free();
-  void concurrent_process_non_strong_references();
-  void concurrent_reset_relocation_set();
-  void pause_verify();
-  void concurrent_select_relocation_set();
-  void pause_relocate_start();
-  void concurrent_relocate();
-
-  void check_out_of_memory();
+  ZDriverPort       _port;
+  ConcurrentGCTimer _gc_timer;
+  ZMinorTracer      _jfr_tracer;
+  size_t            _used_at_start;
 
   void gc(const ZDriverRequest& request);
+  void handle_alloc_stalls() const;
 
 protected:
-  virtual void run_service();
-  virtual void stop_service();
+  virtual void run_thread();
+  virtual void terminate();
 
 public:
-  ZDriver();
+  ZDriverMinor();
 
   bool is_busy() const;
 
   void collect(const ZDriverRequest& request);
+
+  GCTracer* jfr_tracer();
+
+  void set_used_at_start(size_t used);
+  size_t used_at_start() const;
+};
+
+class ZDriverMajor : public ZDriver {
+private:
+  ZDriverPort       _port;
+  ConcurrentGCTimer _gc_timer;
+  ZMajorTracer      _jfr_tracer;
+  size_t            _used_at_start;
+
+  void collect_young(const ZDriverRequest& request);
+
+  void collect_old();
+  void gc(const ZDriverRequest& request);
+  void handle_alloc_stalls() const;
+
+protected:
+  virtual void run_thread();
+  virtual void terminate();
+
+public:
+  ZDriverMajor();
+
+  bool is_busy() const;
+
+  void collect(const ZDriverRequest& request);
+
+  GCTracer* jfr_tracer();
+
+  void set_used_at_start(size_t used);
+  size_t used_at_start() const;
+};
+
+class ZDriverLocker : public StackObj {
+public:
+  ZDriverLocker();
+  ~ZDriverLocker();
+};
+
+class ZDriverUnlocker : public StackObj {
+public:
+  ZDriverUnlocker();
+  ~ZDriverUnlocker();
 };
 
 #endif // SHARE_GC_Z_ZDRIVER_HPP
diff --git a/src/hotspot/share/gc/z/zDriverPort.cpp b/src/hotspot/share/gc/z/zDriverPort.cpp
new file mode 100644
index 00000000000..1dd6c902a8f
--- /dev/null
+++ b/src/hotspot/share/gc/z/zDriverPort.cpp
@@ -0,0 +1,185 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zDriverPort.hpp"
+#include "gc/z/zFuture.inline.hpp"
+#include "gc/z/zList.inline.hpp"
+#include "gc/z/zLock.inline.hpp"
+#include "utilities/debug.hpp"
+
+ZDriverRequest::ZDriverRequest() :
+    ZDriverRequest(GCCause::_no_gc, 0, 0) {}
+
+ZDriverRequest::ZDriverRequest(GCCause::Cause cause, uint young_nworkers, uint old_nworkers) :
+    _cause(cause),
+    _young_nworkers(young_nworkers),
+    _old_nworkers(old_nworkers) {}
+
+bool ZDriverRequest::operator==(const ZDriverRequest& other) const {
+  return _cause == other._cause;
+}
+
+GCCause::Cause ZDriverRequest::cause() const {
+  return _cause;
+}
+
+uint ZDriverRequest::young_nworkers() const {
+  return _young_nworkers;
+}
+
+uint ZDriverRequest::old_nworkers() const {
+  return _old_nworkers;
+}
+
+class ZDriverPortEntry {
+  friend class ZList<ZDriverPortEntry>;
+
+private:
+  const ZDriverRequest        _message;
+  uint64_t                    _seqnum;
+  ZFuture<ZDriverRequest>     _result;
+  ZListNode<ZDriverPortEntry> _node;
+
+public:
+  ZDriverPortEntry(const ZDriverRequest& message) :
+      _message(message),
+      _seqnum(0) {}
+
+  void set_seqnum(uint64_t seqnum) {
+    _seqnum = seqnum;
+  }
+
+  uint64_t seqnum() const {
+    return _seqnum;
+  }
+
+  ZDriverRequest message() const {
+    return _message;
+  }
+
+  void wait() {
+    const ZDriverRequest message = _result.get();
+    assert(message == _message, "Message mismatch");
+  }
+
+  void satisfy(const ZDriverRequest& message) {
+    _result.set(message);
+  }
+};
+
+ZDriverPort::ZDriverPort() :
+    _lock(),
+    _has_message(false),
+    _seqnum(0),
+    _queue() {}
+
+bool ZDriverPort::is_busy() const {
+  ZLocker<ZConditionLock> locker(&_lock);
+  return _has_message;
+}
+
+void ZDriverPort::send_sync(const ZDriverRequest& message) {
+  ZDriverPortEntry entry(message);
+
+  {
+    // Enqueue message
+    ZLocker<ZConditionLock> locker(&_lock);
+    entry.set_seqnum(_seqnum);
+    _queue.insert_last(&entry);
+    _lock.notify();
+  }
+
+  // Wait for completion
+  entry.wait();
+
+  {
+    // Guard deletion of underlying semaphore. This is a workaround for a
+    // bug in sem_post() in glibc < 2.21, where it's not safe to destroy
+    // the semaphore immediately after returning from sem_wait(). The
+    // reason is that sem_post() can touch the semaphore after a waiting
+    // thread have returned from sem_wait(). To avoid this race we are
+    // forcing the waiting thread to acquire/release the lock held by the
+    // posting thread. https://sourceware.org/bugzilla/show_bug.cgi?id=12674
+    ZLocker<ZConditionLock> locker(&_lock);
+  }
+}
+
+void ZDriverPort::send_async(const ZDriverRequest& message) {
+  ZLocker<ZConditionLock> locker(&_lock);
+  if (!_has_message) {
+    // Post message
+    _message = message;
+    _has_message = true;
+    _lock.notify();
+  }
+}
+
+ZDriverRequest ZDriverPort::receive() {
+  ZLocker<ZConditionLock> locker(&_lock);
+
+  // Wait for message
+  while (!_has_message && _queue.is_empty()) {
+    _lock.wait();
+  }
+
+  // Increment request sequence number
+  _seqnum++;
+
+  if (!_has_message) {
+    // Message available in the queue
+    _message = _queue.first()->message();
+    _has_message = true;
+  }
+
+  return _message;
+}
+
+void ZDriverPort::ack() {
+  ZLocker<ZConditionLock> locker(&_lock);
+
+  if (!_has_message) {
+    // Nothing to ack
+    return;
+  }
+
+  // Satisfy requests (and duplicates) in queue
+  ZListIterator<ZDriverPortEntry> iter(&_queue);
+  for (ZDriverPortEntry* entry; iter.next(&entry);) {
+    if (entry->message() == _message && entry->seqnum() < _seqnum) {
+      // Dequeue and satisfy request. Note that the dequeue operation must
+      // happen first, since the request will immediately be deallocated
+      // once it has been satisfied.
+      _queue.remove(entry);
+      entry->satisfy(_message);
+    }
+  }
+
+  if (_queue.is_empty()) {
+    // Queue is empty
+    _has_message = false;
+  } else {
+    // Post first message in queue
+    _message = _queue.first()->message();
+  }
+}
diff --git a/src/hotspot/share/gc/z/zDriverPort.hpp b/src/hotspot/share/gc/z/zDriverPort.hpp
new file mode 100644
index 00000000000..6efdc9bf7f4
--- /dev/null
+++ b/src/hotspot/share/gc/z/zDriverPort.hpp
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZDRIVERPORT_HPP
+#define SHARE_GC_Z_ZDRIVERPORT_HPP
+
+#include "gc/shared/gcCause.hpp"
+#include "gc/z/zList.hpp"
+#include "gc/z/zLock.hpp"
+
+class ZDriverPortEntry;
+
+class ZDriverRequest {
+private:
+  GCCause::Cause _cause;
+  uint           _young_nworkers;
+  uint           _old_nworkers;
+
+public:
+  ZDriverRequest();
+  ZDriverRequest(GCCause::Cause cause, uint young_nworkers, uint old_nworkers);
+
+  bool operator==(const ZDriverRequest& other) const;
+
+  GCCause::Cause cause() const;
+  uint young_nworkers() const;
+  uint old_nworkers() const;
+};
+
+class ZDriverPort {
+private:
+  mutable ZConditionLock  _lock;
+  bool                    _has_message;
+  ZDriverRequest          _message;
+  uint64_t                _seqnum;
+  ZList<ZDriverPortEntry> _queue;
+
+public:
+  ZDriverPort();
+
+  bool is_busy() const;
+
+  // For use by sender
+  void send_sync(const ZDriverRequest& request);
+  void send_async(const ZDriverRequest& request);
+
+  // For use by receiver
+  ZDriverRequest receive();
+  void ack();
+};
+
+#endif // SHARE_GC_Z_ZDRIVERPORT_HPP
diff --git a/src/hotspot/share/gc/z/zForwarding.cpp b/src/hotspot/share/gc/z/zForwarding.cpp
index cabd381cbc1..d8a3913806c 100644
--- a/src/hotspot/share/gc/z/zForwarding.cpp
+++ b/src/hotspot/share/gc/z/zForwarding.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,10 +22,15 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
 #include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zCollectedHeap.hpp"
 #include "gc/z/zForwarding.inline.hpp"
+#include "gc/z/zPage.inline.hpp"
 #include "gc/z/zStat.hpp"
 #include "gc/z/zUtils.inline.hpp"
+#include "logging/log.hpp"
+#include "runtime/atomic.hpp"
 #include "utilities/align.hpp"
 
 //
@@ -45,9 +50,42 @@
 // count has become zero (released) or negative one (claimed).
 //
 
-static const ZStatCriticalPhase ZCriticalPhaseRelocationStall("Relocation Stall");
+bool ZForwarding::claim() {
+  return Atomic::cmpxchg(&_claimed, false, true) == false;
+}
 
-bool ZForwarding::retain_page() {
+void ZForwarding::in_place_relocation_start(zoffset relocated_watermark) {
+  _page->log_msg(" In-place reloc start  - relocated to: " PTR_FORMAT, untype(relocated_watermark));
+
+  _in_place = true;
+
+  // Support for ZHeap::is_in checks of from-space objects
+  // in a page that is in-place relocating
+  Atomic::store(&_in_place_thread, Thread::current());
+  _in_place_top_at_start = _page->top();
+}
+
+void ZForwarding::in_place_relocation_finish() {
+  assert(_in_place, "Must be an in-place relocated page");
+
+  _page->log_msg(" In-place reloc finish - top at start: " PTR_FORMAT, untype(_in_place_top_at_start));
+
+  if (_from_age == ZPageAge::old || _to_age != ZPageAge::old) {
+    // Only do this for non-promoted pages, that still need to reset live map.
+    // Done with iterating over the "from-page" view, so can now drop the _livemap.
+    _page->finalize_reset_for_in_place_relocation();
+  }
+
+  // Disable relaxed ZHeap::is_in checks
+  Atomic::store(&_in_place_thread, (Thread*)nullptr);
+}
+
+bool ZForwarding::in_place_relocation_is_below_top_at_start(zoffset offset) const {
+  // Only the relocating thread is allowed to know about the old relocation top.
+  return Atomic::load(&_in_place_thread) == Thread::current() && offset < _in_place_top_at_start;
+}
+
+bool ZForwarding::retain_page(ZRelocateQueue* queue) {
   for (;;) {
     const int32_t ref_count = Atomic::load_acquire(&_ref_count);
 
@@ -58,8 +96,9 @@ bool ZForwarding::retain_page() {
 
     if (ref_count < 0) {
       // Claimed
-      const bool success = wait_page_released();
-      assert(success, "Should always succeed");
+      queue->add_and_wait(this);
+
+      // Released
       return false;
     }
 
@@ -70,7 +109,7 @@ bool ZForwarding::retain_page() {
   }
 }
 
-ZPage* ZForwarding::claim_page() {
+void ZForwarding::in_place_relocation_claim_page() {
   for (;;) {
     const int32_t ref_count = Atomic::load(&_ref_count);
     assert(ref_count > 0, "Invalid state");
@@ -89,7 +128,8 @@ ZPage* ZForwarding::claim_page() {
       }
     }
 
-    return _page;
+    // Done
+    break;
   }
 }
 
@@ -130,22 +170,6 @@ void ZForwarding::release_page() {
   }
 }
 
-bool ZForwarding::wait_page_released() const {
-  if (Atomic::load_acquire(&_ref_count) != 0) {
-    ZStatTimer timer(ZCriticalPhaseRelocationStall);
-    ZLocker<ZConditionLock> locker(&_ref_lock);
-    while (Atomic::load_acquire(&_ref_count) != 0) {
-      if (_ref_abort) {
-        return false;
-      }
-
-      _ref_lock.wait();
-    }
-  }
-
-  return true;
-}
-
 ZPage* ZForwarding::detach_page() {
   // Wait until released
   if (Atomic::load_acquire(&_ref_count) != 0) {
@@ -155,23 +179,198 @@ ZPage* ZForwarding::detach_page() {
     }
   }
 
-  // Detach and return page
-  ZPage* const page = _page;
-  _page = NULL;
-  return page;
+  return _page;
 }
 
-void ZForwarding::abort_page() {
-  ZLocker<ZConditionLock> locker(&_ref_lock);
-  assert(Atomic::load(&_ref_count) > 0, "Invalid state");
-  assert(!_ref_abort, "Invalid state");
-  _ref_abort = true;
-  _ref_lock.notify_all();
+ZPage* ZForwarding::page() {
+  assert(Atomic::load(&_ref_count) != 0, "The page has been released/detached");
+  return _page;
+}
+
+void ZForwarding::mark_done() {
+  Atomic::store(&_done, true);
+}
+
+bool ZForwarding::is_done() const {
+  return Atomic::load(&_done);
+}
+
+//
+// The relocated_remembered_fields are used when the old generation
+// collection is relocating objects, concurrently with the young
+// generation collection's remembered set scanning for the marking.
+//
+// When the OC is relocating objects, the old remembered set bits
+// for the from-space objects need to be moved over to the to-space
+// objects.
+//
+// The YC doesn't want to wait for the OC, so it eagerly helps relocating
+// objects with remembered set bits, so that it can perform marking on the
+// to-space copy of the object fields that are associated with the remembered
+// set bits.
+//
+// This requires some synchronization between the OC and YC, and this is
+// mainly done via the _relocated_remembered_fields_state in each ZForwarding.
+// The values corresponds to:
+//
+// none:      Starting state - neither OC nor YC has stated their intentions
+// published: The OC has completed relocating all objects, and published an array
+//            of all to-space fields that should have a remembered set entry.
+// reject:    The OC relocation of the page happened concurrently with the YC
+//            remset scanning. Two situations:
+//            a) The page had not been released yet: The YC eagerly relocated and
+//            scanned the to-space objects with remset entries.
+//            b) The page had been released: The YC accepts the array published in
+//            (published).
+// accept:    The YC found that the forwarding/page had already been relocated when
+//            the YC started.
+//
+// Central to this logic is the ZRemembered::scan_forwarding function, where
+// the YC tries to "retain" the forwarding/page. If it succeeds it means that
+// the OC has not finished (or maybe not even started) the relocation of all objects.
+//
+// When the YC manages to retaining the page it will bring the state from:
+//  none      -> reject - Started collecting remembered set info
+//  published -> reject - Rejected the OC's remembered set info
+//  reject    -> reject - An earlier YC had already handled the remembered set info
+//  accept    ->        - Invalid state - will not happen
+//
+// When the YC fails to retain the page the state transitions are:
+// none      -> x - The page was relocated before the YC started
+// published -> x - The OC completed relocation before YC visited this forwarding.
+//                  The YC will use the remembered set info collected by the OC.
+// reject    -> x - A previous YC has already handled the remembered set info
+// accept    -> x - See above
+//
+// x is:
+//  reject        - if the relocation finished while the current YC was running
+//  accept        - if the relocation finished before the current YC started
+//
+// Note the subtlety that even though the relocation could released the page
+// and made it non-retainable, the relocation code might not have gotten to
+// the point where the page is removed from the page table. It could also be
+// the case that the relocated page became in-place relocated, and we therefore
+// shouldn't be scanning it this YC.
+//
+// The (reject) state is the "dangerous" state, where both OC and YC work on
+// the same forwarding/page somewhat concurrently. While (accept) denotes that
+// that the entire relocation of a page (including freeing/reusing it) was
+// completed before the current YC started.
+//
+// After all remset entries of relocated objects have been scanned, the code
+// proceeds to visit all pages in the page table, to scan all pages not part
+// of the OC relocation set. Pages with virtual addresses that doesn't match
+// any of the once in the OC relocation set will be visited. Pages with
+// virtual address that *do* have a corresponding forwarding entry has two
+// cases:
+//
+// a) The forwarding entry is marked with (reject). This means that the
+//    corresponding page is guaranteed to be one that has been relocated by the
+//    current OC during the active YC. Any remset entry is guaranteed to have
+//    already been scanned by the scan_forwarding code.
+//
+// b) The forwarding entry is marked with (accept). This means that the page was
+//    *not* created by the OC relocation during this YC, which means that the
+//    page must be scanned.
+//
+
+void ZForwarding::relocated_remembered_fields_after_relocate() {
+  assert(from_age() == ZPageAge::old, "Only old pages have remsets");
+
+  _relocated_remembered_fields_publish_young_seqnum = ZGeneration::young()->seqnum();
+
+  if (ZGeneration::young()->is_phase_mark()) {
+    relocated_remembered_fields_publish();
+  }
+}
+
+void ZForwarding::relocated_remembered_fields_publish() {
+  // The OC has relocated all objects and collected all fields that
+  // used to have remembered set entries. Now publish the fields to
+  // the YC.
+
+  const ZPublishState res = Atomic::cmpxchg(&_relocated_remembered_fields_state, ZPublishState::none, ZPublishState::published);
+
+  // none:      OK to publish
+  // published: Not possible - this operation makes this transition
+  // reject:    YC started scanning the "from" page concurrently and rejects the fields
+  //            the OC collected.
+  // accept:    YC accepted the fields published by this function - not possible
+  //            because they weren't published before the CAS above
+
+  if (res == ZPublishState::none) {
+    // fields were successfully published
+    log_debug(gc, remset)("Forwarding remset published       : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
+
+    return;
+  }
+
+  log_debug(gc, remset)("Forwarding remset discarded       : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
+
+  // reject: YC scans the remset concurrently
+  // accept: YC accepted published remset - not possible, we just atomically published it
+  //         YC failed to retain page - not possible, since the current page is retainable
+  assert(res == ZPublishState::reject, "Unexpected value");
+
+  // YC has rejected the stored values and will (or have already) find them them itself
+  _relocated_remembered_fields_array.clear_and_deallocate();
+}
+
+void ZForwarding::relocated_remembered_fields_notify_concurrent_scan_of() {
+  // Invariant: The page is being retained
+  assert(ZGeneration::young()->is_phase_mark(), "Only called when");
+
+  const ZPublishState res = Atomic::cmpxchg(&_relocated_remembered_fields_state, ZPublishState::none, ZPublishState::reject);
+
+  // none:      OC has not completed relocation
+  // published: OC has completed and published all relocated remembered fields
+  // reject:    A previous YC has already handled the field
+  // accept:    A previous YC has determined that there's no concurrency between
+  //            OC relocation and YC remembered fields scanning - not possible
+  //            since the page has been retained (still being relocated) and
+  //            we are in the process of scanning fields
+
+  if (res == ZPublishState::none) {
+    // Successfully notified and rejected any collected data from the OC
+    log_debug(gc, remset)("Forwarding remset eager           : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
+
+    return;
+  }
+
+  if (res == ZPublishState::published) {
+    // OC relocation already collected and published fields
+
+    // Still notify concurrent scanning and reject the collected data from the OC
+    const ZPublishState res2 = Atomic::cmpxchg(&_relocated_remembered_fields_state, ZPublishState::published, ZPublishState::reject);
+    assert(res2 == ZPublishState::published, "Should not fail");
+
+    log_debug(gc, remset)("Forwarding remset eager and reject: " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
+
+    // The YC rejected the publish fields and is responsible for the array
+    // Eagerly deallocate the memory
+    _relocated_remembered_fields_array.clear_and_deallocate();
+    return;
+  }
+
+  log_debug(gc, remset)("Forwarding remset redundant       : " PTR_FORMAT " " PTR_FORMAT, untype(start()), untype(end()));
+
+  // Previous YC already handled the remembered fields
+  assert(res == ZPublishState::reject, "Unexpected value");
+}
+
+bool ZForwarding::relocated_remembered_fields_published_contains(volatile zpointer* p) {
+  for (volatile zpointer* const elem : _relocated_remembered_fields_array) {
+    if (elem == p) {
+      return true;
+    }
+  }
+
+  return false;
 }
 
 void ZForwarding::verify() const {
   guarantee(_ref_count != 0, "Invalid reference count");
-  guarantee(_page != NULL, "Invalid page");
+  guarantee(_page != nullptr, "Invalid page");
 
   uint32_t live_objects = 0;
   size_t live_bytes = 0;
@@ -198,7 +397,7 @@ void ZForwarding::verify() const {
       guarantee(entry.to_offset() != other.to_offset(), "Duplicate to");
     }
 
-    const uintptr_t to_addr = ZAddress::good(entry.to_offset());
+    const zaddress to_addr = ZOffset::address(to_zoffset(entry.to_offset()));
     const size_t size = ZUtils::object_size(to_addr);
     const size_t aligned_size = align_up(size, _page->object_alignment());
     live_bytes += aligned_size;
@@ -206,5 +405,5 @@ void ZForwarding::verify() const {
   }
 
   // Verify number of live objects and bytes
-  _page->verify_live(live_objects, live_bytes);
+  _page->verify_live(live_objects, live_bytes, _in_place);
 }
diff --git a/src/hotspot/share/gc/z/zForwarding.hpp b/src/hotspot/share/gc/z/zForwarding.hpp
index 8212cb2f81b..a99473322d4 100644
--- a/src/hotspot/share/gc/z/zForwarding.hpp
+++ b/src/hotspot/share/gc/z/zForwarding.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,14 +24,19 @@
 #ifndef SHARE_GC_Z_ZFORWARDING_HPP
 #define SHARE_GC_Z_ZFORWARDING_HPP
 
+#include "gc/z/zArray.hpp"
 #include "gc/z/zAttachedArray.hpp"
 #include "gc/z/zForwardingEntry.hpp"
+#include "gc/z/zGenerationId.hpp"
 #include "gc/z/zLock.hpp"
+#include "gc/z/zPageAge.hpp"
+#include "gc/z/zPageType.hpp"
 #include "gc/z/zVirtualMemory.hpp"
 
 class ObjectClosure;
 class ZForwardingAllocator;
 class ZPage;
+class ZRelocateQueue;
 
 typedef size_t ZForwardingCursor;
 
@@ -39,47 +44,116 @@ class ZForwarding {
   friend class VMStructs;
   friend class ZForwardingTest;
 
+  enum class ZPublishState : int8_t {
+    none,      // No publishing done yet
+    published, // OC published remset field info, which YC will reject or accept
+    reject,    // YC remset scanning accepted OC published remset field info
+    accept     // YC remset scanning rejected OC published remset field info
+  };
+
 private:
   typedef ZAttachedArray<ZForwarding, ZForwardingEntry> AttachedArray;
+  typedef ZArray<volatile zpointer*> PointerArray;
 
   const ZVirtualMemory   _virtual;
   const size_t           _object_alignment_shift;
   const AttachedArray    _entries;
-  ZPage*                 _page;
+  ZPage* const           _page;
+  ZPageAge               _from_age;
+  ZPageAge               _to_age;
+  volatile bool          _claimed;
   mutable ZConditionLock _ref_lock;
   volatile int32_t       _ref_count;
-  bool                   _ref_abort;
+  volatile bool          _done;
+
+  // Relocated remembered set fields support
+  volatile ZPublishState _relocated_remembered_fields_state;
+  PointerArray           _relocated_remembered_fields_array;
+  uint32_t               _relocated_remembered_fields_publish_young_seqnum;
+
+  // In-place relocation support
   bool                   _in_place;
+  zoffset_end            _in_place_top_at_start;
+
+  // Debugging
+  volatile Thread*       _in_place_thread;
 
   ZForwardingEntry* entries() const;
   ZForwardingEntry at(ZForwardingCursor* cursor) const;
   ZForwardingEntry first(uintptr_t from_index, ZForwardingCursor* cursor) const;
   ZForwardingEntry next(ZForwardingCursor* cursor) const;
 
-  ZForwarding(ZPage* page, size_t nentries);
+  template <typename Function>
+  void object_iterate_forwarded_via_livemap(Function function);
+
+  ZForwarding(ZPage* page, ZPageAge to_age, size_t nentries);
 
 public:
   static uint32_t nentries(const ZPage* page);
-  static ZForwarding* alloc(ZForwardingAllocator* allocator, ZPage* page);
+  static ZForwarding* alloc(ZForwardingAllocator* allocator, ZPage* page, ZPageAge to_age);
 
-  uint8_t type() const;
-  uintptr_t start() const;
+  ZPageType type() const;
+  ZPageAge from_age() const;
+  ZPageAge to_age() const;
+  zoffset start() const;
+  zoffset_end end() const;
   size_t size() const;
   size_t object_alignment_shift() const;
-  void object_iterate(ObjectClosure *cl);
 
-  bool retain_page();
-  ZPage* claim_page();
+  bool is_promotion() const;
+
+  // Visit from-objects
+  template <typename Function>
+  void object_iterate(Function function);
+
+  template <typename Function>
+  void address_unsafe_iterate_via_table(Function function);
+
+  // Visit to-objects
+  template <typename Function>
+  void object_iterate_forwarded(Function function);
+
+  template <typename Function>
+  void object_iterate_forwarded_via_table(Function function);
+
+  template <typename Function>
+  void oops_do_in_forwarded(Function function);
+
+  template <typename Function>
+  void oops_do_in_forwarded_via_table(Function function);
+
+  bool claim();
+
+  // In-place relocation support
+  bool in_place_relocation() const;
+  void in_place_relocation_claim_page();
+  void in_place_relocation_start(zoffset relocated_watermark);
+  void in_place_relocation_finish();
+  bool in_place_relocation_is_below_top_at_start(zoffset addr) const;
+
+  bool retain_page(ZRelocateQueue* queue);
   void release_page();
-  bool wait_page_released() const;
-  ZPage* detach_page();
-  void abort_page();
 
-  void set_in_place();
-  bool in_place() const;
+  ZPage* detach_page();
+  ZPage* page();
+
+  void mark_done();
+  bool is_done() const;
+
+  zaddress find(zaddress_unsafe addr);
 
   ZForwardingEntry find(uintptr_t from_index, ZForwardingCursor* cursor) const;
-  uintptr_t insert(uintptr_t from_index, uintptr_t to_offset, ZForwardingCursor* cursor);
+  zoffset insert(uintptr_t from_index, zoffset to_offset, ZForwardingCursor* cursor);
+
+  // Relocated remembered set fields support
+  void relocated_remembered_fields_register(volatile zpointer* p);
+  void relocated_remembered_fields_after_relocate();
+  void relocated_remembered_fields_publish();
+  void relocated_remembered_fields_notify_concurrent_scan_of();
+  bool relocated_remembered_fields_is_concurrently_scanned() const;
+  template <typename Function>
+  void relocated_remembered_fields_apply_to_published(Function function);
+  bool relocated_remembered_fields_published_contains(volatile zpointer* p);
 
   void verify() const;
 };
diff --git a/src/hotspot/share/gc/z/zForwarding.inline.hpp b/src/hotspot/share/gc/z/zForwarding.inline.hpp
index cff6d7a905c..1afd3ac684b 100644
--- a/src/hotspot/share/gc/z/zForwarding.inline.hpp
+++ b/src/hotspot/share/gc/z/zForwarding.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,12 +26,15 @@
 
 #include "gc/z/zForwarding.hpp"
 
+#include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zAttachedArray.inline.hpp"
 #include "gc/z/zForwardingAllocator.inline.hpp"
 #include "gc/z/zHash.inline.hpp"
 #include "gc/z/zHeap.hpp"
+#include "gc/z/zIterator.inline.hpp"
 #include "gc/z/zLock.inline.hpp"
 #include "gc/z/zPage.inline.hpp"
+#include "gc/z/zUtils.inline.hpp"
 #include "gc/z/zVirtualMemory.inline.hpp"
 #include "runtime/atomic.hpp"
 #include "utilities/debug.hpp"
@@ -47,30 +50,50 @@ inline uint32_t ZForwarding::nentries(const ZPage* page) {
   return round_up_power_of_2(page->live_objects() * 2);
 }
 
-inline ZForwarding* ZForwarding::alloc(ZForwardingAllocator* allocator, ZPage* page) {
+inline ZForwarding* ZForwarding::alloc(ZForwardingAllocator* allocator, ZPage* page, ZPageAge to_age) {
   const size_t nentries = ZForwarding::nentries(page);
   void* const addr = AttachedArray::alloc(allocator, nentries);
-  return ::new (addr) ZForwarding(page, nentries);
+  return ::new (addr) ZForwarding(page, to_age, nentries);
 }
 
-inline ZForwarding::ZForwarding(ZPage* page, size_t nentries) :
+inline ZForwarding::ZForwarding(ZPage* page, ZPageAge to_age, size_t nentries) :
     _virtual(page->virtual_memory()),
     _object_alignment_shift(page->object_alignment_shift()),
     _entries(nentries),
     _page(page),
+    _from_age(page->age()),
+    _to_age(to_age),
+    _claimed(false),
     _ref_lock(),
     _ref_count(1),
-    _ref_abort(false),
-    _in_place(false) {}
+    _done(false),
+    _relocated_remembered_fields_state(ZPublishState::none),
+    _relocated_remembered_fields_array(),
+    _relocated_remembered_fields_publish_young_seqnum(0),
+    _in_place(false),
+    _in_place_top_at_start(),
+    _in_place_thread(nullptr) {}
 
-inline uint8_t ZForwarding::type() const {
+inline ZPageType ZForwarding::type() const {
   return _page->type();
 }
 
-inline uintptr_t ZForwarding::start() const {
+inline ZPageAge ZForwarding::from_age() const {
+  return _from_age;
+}
+
+inline ZPageAge ZForwarding::to_age() const {
+  return _to_age;
+}
+
+inline zoffset ZForwarding::start() const {
   return _virtual.start();
 }
 
+inline zoffset_end ZForwarding::end() const {
+  return _virtual.end();
+}
+
 inline size_t ZForwarding::size() const {
   return _virtual.size();
 }
@@ -79,15 +102,96 @@ inline size_t ZForwarding::object_alignment_shift() const {
   return _object_alignment_shift;
 }
 
-inline void ZForwarding::object_iterate(ObjectClosure *cl) {
-  return _page->object_iterate(cl);
+inline bool ZForwarding::is_promotion() const {
+  return _from_age != ZPageAge::old &&
+         _to_age == ZPageAge::old;
 }
 
-inline void ZForwarding::set_in_place() {
-  _in_place = true;
+template <typename Function>
+inline void ZForwarding::object_iterate(Function function) {
+  ZObjectClosure<Function> cl(function);
+  _page->object_iterate(function);
 }
 
-inline bool ZForwarding::in_place() const {
+template <typename Function>
+inline void ZForwarding::address_unsafe_iterate_via_table(Function function) {
+  for (ZForwardingCursor i = 0; i < _entries.length(); i++) {
+    const ZForwardingEntry entry = at(&i);
+    if (!entry.populated()) {
+      // Skip empty entries
+      continue;
+    }
+
+    // Find to-object
+
+    const zoffset from_offset = start() + (entry.from_index() << object_alignment_shift());
+    const zaddress_unsafe from_addr = ZOffset::address_unsafe(from_offset);
+
+    // Apply function
+    function(from_addr);
+  }
+}
+
+template <typename Function>
+inline void ZForwarding::object_iterate_forwarded_via_livemap(Function function) {
+  assert(!in_place_relocation(), "Not allowed to use livemap iteration");
+
+  object_iterate([&](oop obj) {
+    // Find to-object
+    const zaddress_unsafe from_addr = to_zaddress_unsafe(obj);
+    const zaddress to_addr = this->find(from_addr);
+    const oop to_obj = to_oop(to_addr);
+
+    // Apply function
+    function(to_obj);
+  });
+}
+
+template <typename Function>
+inline void ZForwarding::object_iterate_forwarded_via_table(Function function) {
+  for (ZForwardingCursor i = 0; i < _entries.length(); i++) {
+    const ZForwardingEntry entry = at(&i);
+    if (!entry.populated()) {
+      // Skip empty entries
+      continue;
+    }
+
+    // Find to-object
+    const zoffset to_offset = to_zoffset(entry.to_offset());
+    const zaddress to_addr = ZOffset::address(to_offset);
+    const oop to_obj = to_oop(to_addr);
+
+    // Apply function
+    function(to_obj);
+  }
+}
+
+template <typename Function>
+inline void ZForwarding::object_iterate_forwarded(Function function) {
+  if (in_place_relocation()) {
+    // The original objects are not available anymore, can't use the livemap
+    object_iterate_forwarded_via_table(function);
+  } else {
+    object_iterate_forwarded_via_livemap(function);
+  }
+}
+
+template <typename Function>
+void ZForwarding::oops_do_in_forwarded(Function function) {
+  object_iterate_forwarded([&](oop to_obj) {
+    ZIterator::basic_oop_iterate_safe(to_obj, function);
+  });
+}
+
+template <typename Function>
+void ZForwarding::oops_do_in_forwarded_via_table(Function function) {
+  object_iterate_forwarded_via_table([&](oop to_obj) {
+    ZIterator::basic_oop_iterate_safe(to_obj, function);
+  });
+}
+
+inline bool ZForwarding::in_place_relocation() const {
+  assert(Atomic::load(&_ref_count) != 0, "The page has been released/detached");
   return _in_place;
 }
 
@@ -114,6 +218,13 @@ inline ZForwardingEntry ZForwarding::next(ZForwardingCursor* cursor) const {
   return at(cursor);
 }
 
+inline zaddress ZForwarding::find(zaddress_unsafe addr) {
+  const uintptr_t from_index = (ZAddress::offset(addr) - start()) >> object_alignment_shift();
+  ZForwardingCursor cursor;
+  const ZForwardingEntry entry = find(from_index, &cursor);
+  return entry.populated() ? ZOffset::address(to_zoffset(entry.to_offset())) : zaddress::null;
+}
+
 inline ZForwardingEntry ZForwarding::find(uintptr_t from_index, ZForwardingCursor* cursor) const {
   // Reading entries in the table races with the atomic CAS done for
   // insertion into the table. This is safe because each entry is at
@@ -132,8 +243,8 @@ inline ZForwardingEntry ZForwarding::find(uintptr_t from_index, ZForwardingCurso
   return entry;
 }
 
-inline uintptr_t ZForwarding::insert(uintptr_t from_index, uintptr_t to_offset, ZForwardingCursor* cursor) {
-  const ZForwardingEntry new_entry(from_index, to_offset);
+inline zoffset ZForwarding::insert(uintptr_t from_index, zoffset to_offset, ZForwardingCursor* cursor) {
+  const ZForwardingEntry new_entry(from_index, untype(to_offset));
   const ZForwardingEntry old_entry; // Empty
 
   // Make sure that object copy is finished
@@ -152,7 +263,7 @@ inline uintptr_t ZForwarding::insert(uintptr_t from_index, uintptr_t to_offset,
     while (entry.populated()) {
       if (entry.from_index() == from_index) {
         // Match found, return already inserted address
-        return entry.to_offset();
+        return to_zoffset(entry.to_offset());
       }
 
       entry = next(cursor);
@@ -160,4 +271,75 @@ inline uintptr_t ZForwarding::insert(uintptr_t from_index, uintptr_t to_offset,
   }
 }
 
+inline void ZForwarding::relocated_remembered_fields_register(volatile zpointer* p) {
+  // Invariant: Page is being retained
+  assert(ZGeneration::young()->is_phase_mark(), "Only called when");
+
+  const ZPublishState res = Atomic::load(&_relocated_remembered_fields_state);
+
+  // none:      Gather remembered fields
+  // published: Have already published fields - not possible since they haven't been
+  //            collected yet
+  // reject:    YC rejected fields collected by the OC
+  // accept:    YC has marked that there's no more concurrent scanning of relocated
+  //            fields - not possible since this code is still relocating objects
+
+  if (res == ZPublishState::none) {
+    _relocated_remembered_fields_array.push(p);
+    return;
+  }
+
+  assert(res == ZPublishState::reject, "Unexpected value");
+}
+
+// Returns true iff the page is being (or about to be) relocated by the OC
+// while the YC gathered the remembered fields of the "from" page.
+inline bool ZForwarding::relocated_remembered_fields_is_concurrently_scanned() const {
+  return Atomic::load(&_relocated_remembered_fields_state) == ZPublishState::reject;
+}
+
+template <typename Function>
+inline void ZForwarding::relocated_remembered_fields_apply_to_published(Function function) {
+  // Invariant: Page is not being retained
+  assert(ZGeneration::young()->is_phase_mark(), "Only called when");
+
+  const ZPublishState res = Atomic::load_acquire(&_relocated_remembered_fields_state);
+
+  // none:      Nothing published - page had already been relocated before YC started
+  // published: OC relocated and published relocated remembered fields
+  // reject:    A previous YC concurrently scanned relocated remembered fields of the "from" page
+  // accept:    A previous YC marked that it didn't do (reject)
+
+  if (res == ZPublishState::published) {
+    log_debug(gc, remset)("Forwarding remset accept          : " PTR_FORMAT " " PTR_FORMAT " (" PTR_FORMAT ", %s)",
+        untype(start()), untype(end()), p2i(this), Thread::current()->name());
+
+    // OC published relocated remembered fields
+    ZArrayIterator<volatile zpointer*> iter(&_relocated_remembered_fields_array);
+    for (volatile zpointer* to_field_addr; iter.next(&to_field_addr);) {
+      function(to_field_addr);
+    }
+
+    // YC responsible for the array - eagerly deallocate
+    _relocated_remembered_fields_array.clear_and_deallocate();
+  }
+
+  assert(_relocated_remembered_fields_publish_young_seqnum != 0, "Must have been set");
+  if (_relocated_remembered_fields_publish_young_seqnum == ZGeneration::young()->seqnum()) {
+    log_debug(gc, remset)("scan_forwarding failed retain unsafe " PTR_FORMAT, untype(start()));
+    // The page was relocated concurrently with the current young generation
+    // collection. Mark that it is unsafe (and unnecessary) to call scan_page
+    // on the page in the page table.
+    assert(res != ZPublishState::accept, "Unexpected");
+    Atomic::store(&_relocated_remembered_fields_state, ZPublishState::reject);
+  } else {
+    log_debug(gc, remset)("scan_forwarding failed retain safe " PTR_FORMAT, untype(start()));
+    // Guaranteed that the page was fully relocated and removed from page table.
+    // Because of this we can signal to scan_page that any page found in page table
+    // of the same slot as the current forwarding is a page that is safe to scan,
+    // and in fact must be scanned.
+    Atomic::store(&_relocated_remembered_fields_state, ZPublishState::accept);
+  }
+}
+
 #endif // SHARE_GC_Z_ZFORWARDING_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zForwardingAllocator.cpp b/src/hotspot/share/gc/z/zForwardingAllocator.cpp
index 6550259966b..814d4b83f95 100644
--- a/src/hotspot/share/gc/z/zForwardingAllocator.cpp
+++ b/src/hotspot/share/gc/z/zForwardingAllocator.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,9 +26,9 @@
 #include "memory/allocation.inline.hpp"
 
 ZForwardingAllocator::ZForwardingAllocator() :
-    _start(NULL),
-    _end(NULL),
-    _top(NULL) {}
+    _start(nullptr),
+    _end(nullptr),
+    _top(nullptr) {}
 
 ZForwardingAllocator::~ZForwardingAllocator() {
   FREE_C_HEAP_ARRAY(char, _start);
diff --git a/src/hotspot/share/gc/z/zForwardingEntry.hpp b/src/hotspot/share/gc/z/zForwardingEntry.hpp
index eb064ecdf6e..20a81b6e05c 100644
--- a/src/hotspot/share/gc/z/zForwardingEntry.hpp
+++ b/src/hotspot/share/gc/z/zForwardingEntry.hpp
@@ -27,6 +27,7 @@
 #include "gc/z/zBitField.hpp"
 #include "memory/allocation.hpp"
 #include "metaprogramming/primitiveConversions.hpp"
+
 #include <type_traits>
 
 //
diff --git a/src/hotspot/share/gc/z/zForwardingTable.hpp b/src/hotspot/share/gc/z/zForwardingTable.hpp
index ad57646373b..697756a38fa 100644
--- a/src/hotspot/share/gc/z/zForwardingTable.hpp
+++ b/src/hotspot/share/gc/z/zForwardingTable.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,19 +25,23 @@
 #define SHARE_GC_Z_ZFORWARDINGTABLE_HPP
 
 #include "gc/z/zGranuleMap.hpp"
+#include "gc/z/zIndexDistributor.hpp"
 
 class ZForwarding;
 
 class ZForwardingTable {
+  friend class ZRemsetTableIterator;
   friend class VMStructs;
 
 private:
   ZGranuleMap<ZForwarding*> _map;
 
+  ZForwarding* at(size_t index) const;
+
 public:
   ZForwardingTable();
 
-  ZForwarding* get(uintptr_t addr) const;
+  ZForwarding* get(zaddress_unsafe addr) const;
 
   void insert(ZForwarding* forwarding);
   void remove(ZForwarding* forwarding);
diff --git a/src/hotspot/share/gc/z/zForwardingTable.inline.hpp b/src/hotspot/share/gc/z/zForwardingTable.inline.hpp
index 96432474094..543a61e9560 100644
--- a/src/hotspot/share/gc/z/zForwardingTable.inline.hpp
+++ b/src/hotspot/share/gc/z/zForwardingTable.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,30 +30,35 @@
 #include "gc/z/zForwarding.inline.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zGranuleMap.inline.hpp"
+#include "gc/z/zIndexDistributor.inline.hpp"
 #include "utilities/debug.hpp"
 
 inline ZForwardingTable::ZForwardingTable() :
     _map(ZAddressOffsetMax) {}
 
-inline ZForwarding* ZForwardingTable::get(uintptr_t addr) const {
-  assert(!ZAddress::is_null(addr), "Invalid address");
+inline ZForwarding* ZForwardingTable::at(size_t index) const {
+  return _map.at(index);
+}
+
+inline ZForwarding* ZForwardingTable::get(zaddress_unsafe addr) const {
+  assert(!is_null(addr), "Invalid address");
   return _map.get(ZAddress::offset(addr));
 }
 
 inline void ZForwardingTable::insert(ZForwarding* forwarding) {
-  const uintptr_t offset = forwarding->start();
+  const zoffset offset = forwarding->start();
   const size_t size = forwarding->size();
 
-  assert(_map.get(offset) == NULL, "Invalid entry");
+  assert(_map.get(offset) == nullptr, "Invalid entry");
   _map.put(offset, size, forwarding);
 }
 
 inline void ZForwardingTable::remove(ZForwarding* forwarding) {
-  const uintptr_t offset = forwarding->start();
+  const zoffset offset = forwarding->start();
   const size_t size = forwarding->size();
 
   assert(_map.get(offset) == forwarding, "Invalid entry");
-  _map.put(offset, size, NULL);
+  _map.put(offset, size, nullptr);
 }
 
 #endif // SHARE_GC_Z_ZFORWARDINGTABLE_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zGCIdPrinter.cpp b/src/hotspot/share/gc/z/zGCIdPrinter.cpp
new file mode 100644
index 00000000000..ed36d1b9bcf
--- /dev/null
+++ b/src/hotspot/share/gc/z/zGCIdPrinter.cpp
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zGCIdPrinter.hpp"
+#include "include/jvm.h"
+
+ZGCIdPrinter* ZGCIdPrinter::_instance;
+
+void ZGCIdPrinter::initialize() {
+  _instance = new ZGCIdPrinter();
+  GCId::set_printer(_instance);
+}
+
+int ZGCIdPrinter::print_gc_id_unchecked(uint gc_id, char* buf, size_t len) {
+  if (gc_id == _minor_gc_id) {
+    // Minor collections are always tagged with 'y'
+    return jio_snprintf(buf, len, "GC(%u) y: ", gc_id);
+  }
+
+  if (gc_id == _major_gc_id) {
+    // Major collections are either tagged with 'Y' or 'O',
+    // this is controlled by _major_tag.
+    return jio_snprintf(buf, len, "GC(%u) %c: ", gc_id, _major_tag);
+  }
+
+  // The initial log for each GC should be untagged this
+  // is handled by not yet having set the current GC id
+  // for that collection and thus falling through to here.
+  return jio_snprintf(buf, len, "GC(%u) ", gc_id);
+}
+
+size_t ZGCIdPrinter::print_gc_id(uint gc_id, char* buf, size_t len) {
+  const int ret = print_gc_id_unchecked(gc_id, buf, len);
+  assert(ret > 0, "Failed to print prefix. Log buffer too small?");
+  return (size_t)ret;
+}
+
+ZGCIdPrinter::ZGCIdPrinter() :
+    _minor_gc_id(GCId::undefined()),
+    _major_gc_id(GCId::undefined()),
+    _major_tag('-') { }
+
+void ZGCIdPrinter::set_minor_gc_id(uint id) {
+  _minor_gc_id = id;
+}
+
+void ZGCIdPrinter::set_major_gc_id(uint id) {
+ _major_gc_id = id;
+}
+
+void ZGCIdPrinter::set_major_tag(char tag) {
+  _major_tag = tag;
+}
+
+ZGCIdMinor::ZGCIdMinor(uint gc_id) {
+  ZGCIdPrinter::_instance->set_minor_gc_id(gc_id);
+}
+
+ZGCIdMinor::~ZGCIdMinor() {
+  ZGCIdPrinter::_instance->set_minor_gc_id(GCId::undefined());
+}
+
+ZGCIdMajor::ZGCIdMajor(uint gc_id, char tag) {
+  ZGCIdPrinter::_instance->set_major_gc_id(gc_id);
+  ZGCIdPrinter::_instance->set_major_tag(tag);
+}
+
+ZGCIdMajor::~ZGCIdMajor() {
+  ZGCIdPrinter::_instance->set_major_gc_id(GCId::undefined());
+  ZGCIdPrinter::_instance->set_major_tag('-');
+}
diff --git a/src/hotspot/share/gc/z/zGCIdPrinter.hpp b/src/hotspot/share/gc/z/zGCIdPrinter.hpp
new file mode 100644
index 00000000000..38bebf7ab67
--- /dev/null
+++ b/src/hotspot/share/gc/z/zGCIdPrinter.hpp
@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZGCIDPRINTER_HPP
+#define SHARE_GC_Z_ZGCIDPRINTER_HPP
+
+#include "gc/shared/gcId.hpp"
+#include "memory/allocation.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+class ZGCIdPrinter : public GCIdPrinter {
+  friend class ZGCIdMajor;
+  friend class ZGCIdMinor;
+
+private:
+  static ZGCIdPrinter* _instance;
+
+  uint _minor_gc_id;
+  uint _major_gc_id;
+  char _major_tag;
+
+  ZGCIdPrinter();
+
+  void set_minor_gc_id(uint id);
+  void set_major_gc_id(uint id);
+  void set_major_tag(char tag);
+
+  int print_gc_id_unchecked(uint gc_id, char *buf, size_t len);
+  size_t print_gc_id(uint gc_id, char *buf, size_t len) override;
+
+public:
+  static void initialize();
+};
+
+class ZGCIdMinor : public StackObj {
+public:
+  ZGCIdMinor(uint gc_id);
+  ~ZGCIdMinor();
+};
+
+class ZGCIdMajor : public StackObj {
+public:
+  ZGCIdMajor(uint gc_id, char tag);
+  ~ZGCIdMajor();
+};
+
+#endif // SHARE_GC_Z_ZGCIDPRINTER_HPP
diff --git a/src/hotspot/share/gc/z/zGeneration.cpp b/src/hotspot/share/gc/z/zGeneration.cpp
new file mode 100644
index 00000000000..2fd231484e0
--- /dev/null
+++ b/src/hotspot/share/gc/z/zGeneration.cpp
@@ -0,0 +1,1486 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "classfile/classLoaderDataGraph.hpp"
+#include "code/nmethod.hpp"
+#include "gc/shared/gcLocker.hpp"
+#include "gc/shared/gcVMOperations.hpp"
+#include "gc/shared/isGCActiveMark.hpp"
+#include "gc/shared/suspendibleThreadSet.hpp"
+#include "gc/z/zAllocator.inline.hpp"
+#include "gc/z/zBarrierSet.hpp"
+#include "gc/z/zBarrierSetAssembler.hpp"
+#include "gc/z/zBarrierSetNMethod.hpp"
+#include "gc/z/zBreakpoint.hpp"
+#include "gc/z/zCollectedHeap.hpp"
+#include "gc/z/zDriver.hpp"
+#include "gc/z/zForwarding.hpp"
+#include "gc/z/zForwardingTable.inline.hpp"
+#include "gc/z/zGeneration.inline.hpp"
+#include "gc/z/zHeap.inline.hpp"
+#include "gc/z/zJNICritical.hpp"
+#include "gc/z/zMark.inline.hpp"
+#include "gc/z/zPageAllocator.hpp"
+#include "gc/z/zRelocationSet.inline.hpp"
+#include "gc/z/zRelocationSetSelector.inline.hpp"
+#include "gc/z/zRemembered.hpp"
+#include "gc/z/zRootsIterator.hpp"
+#include "gc/z/zStat.hpp"
+#include "gc/z/zTask.hpp"
+#include "gc/z/zUncoloredRoot.inline.hpp"
+#include "gc/z/zVerify.hpp"
+#include "gc/z/zWorkers.hpp"
+#include "logging/log.hpp"
+#include "memory/universe.hpp"
+#include "prims/jvmtiTagMap.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/continuation.hpp"
+#include "runtime/handshake.hpp"
+#include "runtime/safepoint.hpp"
+#include "runtime/threads.hpp"
+#include "runtime/vmOperations.hpp"
+#include "runtime/vmThread.hpp"
+#include "utilities/debug.hpp"
+#include "utilities/events.hpp"
+
+static const ZStatPhaseGeneration ZPhaseGenerationYoung[] {
+  ZStatPhaseGeneration("Young Generation", ZGenerationId::young),
+  ZStatPhaseGeneration("Young Generation (Promote All)", ZGenerationId::young),
+  ZStatPhaseGeneration("Young Generation (Collect Roots)", ZGenerationId::young),
+  ZStatPhaseGeneration("Young Generation", ZGenerationId::young)
+};
+
+static const ZStatPhaseGeneration ZPhaseGenerationOld("Old Generation", ZGenerationId::old);
+
+static const ZStatPhasePause      ZPhasePauseMarkStartYoung("Pause Mark Start", ZGenerationId::young);
+static const ZStatPhasePause      ZPhasePauseMarkStartYoungAndOld("Pause Mark Start (Major)", ZGenerationId::young);
+static const ZStatPhaseConcurrent ZPhaseConcurrentMarkYoung("Concurrent Mark", ZGenerationId::young);
+static const ZStatPhaseConcurrent ZPhaseConcurrentMarkContinueYoung("Concurrent Mark Continue", ZGenerationId::young);
+static const ZStatPhasePause      ZPhasePauseMarkEndYoung("Pause Mark End", ZGenerationId::young);
+static const ZStatPhaseConcurrent ZPhaseConcurrentMarkFreeYoung("Concurrent Mark Free", ZGenerationId::young);
+static const ZStatPhaseConcurrent ZPhaseConcurrentResetRelocationSetYoung("Concurrent Reset Relocation Set", ZGenerationId::young);
+static const ZStatPhaseConcurrent ZPhaseConcurrentSelectRelocationSetYoung("Concurrent Select Relocation Set", ZGenerationId::young);
+static const ZStatPhasePause      ZPhasePauseRelocateStartYoung("Pause Relocate Start", ZGenerationId::young);
+static const ZStatPhaseConcurrent ZPhaseConcurrentRelocatedYoung("Concurrent Relocate", ZGenerationId::young);
+
+static const ZStatPhaseConcurrent ZPhaseConcurrentMarkOld("Concurrent Mark", ZGenerationId::old);
+static const ZStatPhaseConcurrent ZPhaseConcurrentMarkContinueOld("Concurrent Mark Continue", ZGenerationId::old);
+static const ZStatPhasePause      ZPhasePauseMarkEndOld("Pause Mark End", ZGenerationId::old);
+static const ZStatPhaseConcurrent ZPhaseConcurrentMarkFreeOld("Concurrent Mark Free", ZGenerationId::old);
+static const ZStatPhaseConcurrent ZPhaseConcurrentProcessNonStrongOld("Concurrent Process Non-Strong", ZGenerationId::old);
+static const ZStatPhaseConcurrent ZPhaseConcurrentResetRelocationSetOld("Concurrent Reset Relocation Set", ZGenerationId::old);
+static const ZStatPhaseConcurrent ZPhaseConcurrentSelectRelocationSetOld("Concurrent Select Relocation Set", ZGenerationId::old);
+static const ZStatPhasePause      ZPhasePauseRelocateStartOld("Pause Relocate Start", ZGenerationId::old);
+static const ZStatPhaseConcurrent ZPhaseConcurrentRelocatedOld("Concurrent Relocate", ZGenerationId::old);
+static const ZStatPhaseConcurrent ZPhaseConcurrentRemapRootsOld("Concurrent Remap Roots", ZGenerationId::old);
+
+static const ZStatSubPhase ZSubPhaseConcurrentMarkRootsYoung("Concurrent Mark Roots", ZGenerationId::young);
+static const ZStatSubPhase ZSubPhaseConcurrentMarkFollowYoung("Concurrent Mark Follow", ZGenerationId::young);
+
+static const ZStatSubPhase ZSubPhaseConcurrentMarkRootsOld("Concurrent Mark Roots", ZGenerationId::old);
+static const ZStatSubPhase ZSubPhaseConcurrentMarkFollowOld("Concurrent Mark Follow", ZGenerationId::old);
+static const ZStatSubPhase ZSubPhaseConcurrentRemapRootsColoredOld("Concurrent Remap Roots Colored", ZGenerationId::old);
+static const ZStatSubPhase ZSubPhaseConcurrentRemapRootsUncoloredOld("Concurrent Remap Roots Uncolored", ZGenerationId::old);
+static const ZStatSubPhase ZSubPhaseConcurrentRemapRememberedOld("Concurrent Remap Remembered", ZGenerationId::old);
+
+static const ZStatSampler ZSamplerJavaThreads("System", "Java Threads", ZStatUnitThreads);
+
+ZGenerationYoung* ZGeneration::_young;
+ZGenerationOld*   ZGeneration::_old;
+
+ZGeneration::ZGeneration(ZGenerationId id, ZPageTable* page_table, ZPageAllocator* page_allocator) :
+    _id(id),
+    _page_allocator(page_allocator),
+    _page_table(page_table),
+    _forwarding_table(),
+    _workers(id, &_stat_workers),
+    _mark(this, page_table),
+    _relocate(this),
+    _relocation_set(this),
+    _freed(0),
+    _promoted(0),
+    _compacted(0),
+    _phase(Phase::Relocate),
+    _seqnum(1),
+    _stat_heap(),
+    _stat_cycle(),
+    _stat_workers(),
+    _stat_mark(),
+    _stat_relocation(),
+    _gc_timer(nullptr) {
+}
+
+bool ZGeneration::is_initialized() const {
+  return _mark.is_initialized();
+}
+
+ZWorkers* ZGeneration::workers() {
+  return &_workers;
+}
+
+uint ZGeneration::active_workers() const {
+  return _workers.active_workers();
+}
+
+void ZGeneration::set_active_workers(uint nworkers) {
+  _workers.set_active_workers(nworkers);
+}
+
+void ZGeneration::threads_do(ThreadClosure* tc) const {
+  _workers.threads_do(tc);
+}
+
+void ZGeneration::mark_flush_and_free(Thread* thread) {
+  _mark.flush_and_free(thread);
+}
+
+void ZGeneration::mark_free() {
+   _mark.free();
+}
+
+void ZGeneration::free_empty_pages(ZRelocationSetSelector* selector, int bulk) {
+  // Freeing empty pages in bulk is an optimization to avoid grabbing
+  // the page allocator lock, and trying to satisfy stalled allocations
+  // too frequently.
+  if (selector->should_free_empty_pages(bulk)) {
+    const size_t freed = ZHeap::heap()->free_empty_pages(selector->empty_pages());
+    increase_freed(freed);
+    selector->clear_empty_pages();
+  }
+}
+
+void ZGeneration::flip_age_pages(const ZRelocationSetSelector* selector) {
+  if (is_young()) {
+    _relocate.flip_age_pages(selector->not_selected_small());
+    _relocate.flip_age_pages(selector->not_selected_medium());
+    _relocate.flip_age_pages(selector->not_selected_large());
+  }
+}
+
+static double fragmentation_limit(ZGenerationId generation) {
+  if (generation == ZGenerationId::old) {
+    return ZFragmentationLimit;
+  } else {
+    return ZYoungCompactionLimit;
+  }
+}
+
+void ZGeneration::select_relocation_set(ZGenerationId generation, bool promote_all) {
+  // Register relocatable pages with selector
+  ZRelocationSetSelector selector(fragmentation_limit(generation));
+  {
+    ZGenerationPagesIterator pt_iter(_page_table, _id, _page_allocator);
+    for (ZPage* page; pt_iter.next(&page);) {
+      if (!page->is_relocatable()) {
+        // Not relocatable, don't register
+        // Note that the seqnum can change under our feet here as the page
+        // can be concurrently freed and recycled by a concurrent generation
+        // collection. However this property is stable across such transitions.
+        // If it was not relocatable before recycling, then it won't be
+        // relocatable after it gets recycled either, as the seqnum atomically
+        // becomes allocating for the given generation. The opposite property
+        // also holds: if the page is relocatable, then it can't have been
+        // concurrently freed; if it was re-allocated it would not be
+        // relocatable, and if it was not re-allocated we know that it was
+        // allocated earlier than mark start of the current generation
+        // collection.
+        continue;
+      }
+
+      if (page->is_marked()) {
+        // Register live page
+        selector.register_live_page(page);
+      } else {
+        // Register empty page
+        selector.register_empty_page(page);
+
+        // Reclaim empty pages in bulk
+
+        // An active iterator blocks immediate recycle and delete of pages.
+        // The intent it to allow the code that iterates over the pages to
+        // safely read the properties of the pages without them being changed
+        // by another thread. However, this function both iterates over the
+        // pages AND frees/recycles them. We "yield" the iterator, so that we
+        // can perform immediate recycling (as long as no other thread is
+        // iterating over the pages). The contract is that the pages that are
+        // about to be freed are "owned" by this thread, and no other thread
+        // will change their states.
+        pt_iter.yield([&]() {
+          free_empty_pages(&selector, 64 /* bulk */);
+        });
+      }
+    }
+
+    // Reclaim remaining empty pages
+    free_empty_pages(&selector, 0 /* bulk */);
+  }
+
+  // Select relocation set
+  selector.select();
+
+  // Selecting tenuring threshold must be done after select
+  // which produces the liveness data, but before install,
+  // which consumes the tenuring threshold.
+  if (generation == ZGenerationId::young) {
+    ZGeneration::young()->select_tenuring_threshold(selector.stats(), promote_all);
+  }
+
+  // Install relocation set
+  _relocation_set.install(&selector);
+
+  // Flip age young pages that were not selected
+  flip_age_pages(&selector);
+
+  // Setup forwarding table
+  ZRelocationSetIterator rs_iter(&_relocation_set);
+  for (ZForwarding* forwarding; rs_iter.next(&forwarding);) {
+    _forwarding_table.insert(forwarding);
+  }
+
+  // Update statistics
+  stat_relocation()->at_select_relocation_set(selector.stats());
+  stat_heap()->at_select_relocation_set(selector.stats());
+}
+
+ZRelocationSetParallelIterator ZGeneration::relocation_set_parallel_iterator() {
+  return ZRelocationSetParallelIterator(&_relocation_set);
+}
+
+void ZGeneration::reset_relocation_set() {
+  // Reset forwarding table
+  ZRelocationSetIterator iter(&_relocation_set);
+  for (ZForwarding* forwarding; iter.next(&forwarding);) {
+    _forwarding_table.remove(forwarding);
+  }
+
+  // Reset relocation set
+  _relocation_set.reset(_page_allocator);
+}
+
+void ZGeneration::synchronize_relocation() {
+  _relocate.synchronize();
+}
+
+void ZGeneration::desynchronize_relocation() {
+  _relocate.desynchronize();
+}
+
+void ZGeneration::reset_statistics() {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+  _freed = 0;
+  _promoted = 0;
+  _compacted = 0;
+  _page_allocator->reset_statistics(_id);
+}
+
+ssize_t ZGeneration::freed() const {
+  return _freed;
+}
+
+void ZGeneration::increase_freed(size_t size) {
+  Atomic::add(&_freed, size, memory_order_relaxed);
+}
+
+size_t ZGeneration::promoted() const {
+  return _promoted;
+}
+
+void ZGeneration::increase_promoted(size_t size) {
+  Atomic::add(&_promoted, size, memory_order_relaxed);
+}
+
+size_t ZGeneration::compacted() const {
+  return _compacted;
+}
+
+void ZGeneration::increase_compacted(size_t size) {
+  Atomic::add(&_compacted, size, memory_order_relaxed);
+}
+
+ConcurrentGCTimer* ZGeneration::gc_timer() const {
+  return _gc_timer;
+}
+
+void ZGeneration::set_gc_timer(ConcurrentGCTimer* gc_timer) {
+  assert(_gc_timer == nullptr, "Incorrect scoping");
+  _gc_timer = gc_timer;
+}
+
+void ZGeneration::clear_gc_timer() {
+  assert(_gc_timer != nullptr, "Incorrect scoping");
+  _gc_timer = nullptr;
+}
+
+void ZGeneration::log_phase_switch(Phase from, Phase to) {
+  const char* const str[] = {
+    "Young Mark Start",
+    "Young Mark End",
+    "Young Relocate Start",
+    "Old Mark Start",
+    "Old Mark End",
+    "Old Relocate Start"
+  };
+
+  size_t index = 0;
+
+  if (is_old()) {
+    index += 3;
+  }
+
+  if (to == Phase::Relocate) {
+    index += 2;
+  }
+
+  if (from == Phase::Mark && to == Phase::MarkComplete) {
+    index += 1;
+  }
+
+  assert(index < ARRAY_SIZE(str), "OOB: " SIZE_FORMAT " < " SIZE_FORMAT, index, ARRAY_SIZE(str));
+
+  Events::log_zgc_phase_switch("%-21s %4u", str[index], seqnum());
+}
+
+void ZGeneration::set_phase(Phase new_phase) {
+  log_phase_switch(_phase, new_phase);
+
+  _phase = new_phase;
+}
+
+void ZGeneration::at_collection_start(ConcurrentGCTimer* gc_timer) {
+  set_gc_timer(gc_timer);
+  stat_cycle()->at_start();
+  stat_heap()->at_collection_start(_page_allocator->stats(this));
+  workers()->set_active();
+}
+
+void ZGeneration::at_collection_end() {
+  workers()->set_inactive();
+  stat_cycle()->at_end(stat_workers(), should_record_stats());
+  // The heap at collection end data is gathered at relocate end
+  clear_gc_timer();
+}
+
+const char* ZGeneration::phase_to_string() const {
+  switch (_phase) {
+  case Phase::Mark:
+    return "Mark";
+
+  case Phase::MarkComplete:
+    return "MarkComplete";
+
+  case Phase::Relocate:
+    return "Relocate";
+
+  default:
+    return "Unknown";
+  }
+}
+
+class VM_ZOperation : public VM_Operation {
+private:
+  const uint _gc_id;
+  bool       _success;
+
+public:
+  VM_ZOperation() :
+      _gc_id(GCId::current()),
+      _success(false) {}
+
+  virtual bool block_jni_critical() const {
+    // Blocking JNI critical regions is needed in operations where we change
+    // the bad mask or move objects. Changing the bad mask will invalidate all
+    // oops, which makes it conceptually the same thing as moving all objects.
+    return false;
+  }
+
+  virtual bool skip_thread_oop_barriers() const {
+    return true;
+  }
+
+  virtual bool do_operation() = 0;
+
+  virtual bool doit_prologue() {
+    Heap_lock->lock();
+    return true;
+  }
+
+  virtual void doit() {
+    // Setup GC id and active marker
+    GCIdMark gc_id_mark(_gc_id);
+    IsGCActiveMark gc_active_mark;
+
+    // Verify before operation
+    ZVerify::before_zoperation();
+
+    // Execute operation
+    _success = do_operation();
+
+    // Update statistics
+    ZStatSample(ZSamplerJavaThreads, Threads::number_of_threads());
+  }
+
+  virtual void doit_epilogue() {
+    Heap_lock->unlock();
+  }
+
+  bool success() const {
+    return _success;
+  }
+
+  bool pause() {
+    if (block_jni_critical()) {
+      ZJNICritical::block();
+    }
+
+    VMThread::execute(this);
+
+    if (block_jni_critical()) {
+      ZJNICritical::unblock();
+    }
+
+    return _success;
+  }
+};
+
+ZYoungTypeSetter::ZYoungTypeSetter(ZYoungType type) {
+  assert(ZGeneration::young()->_active_type == ZYoungType::none, "Invalid type");
+  ZGeneration::young()->_active_type = type;
+}
+
+ZYoungTypeSetter::~ZYoungTypeSetter() {
+  assert(ZGeneration::young()->_active_type != ZYoungType::none, "Invalid type");
+  ZGeneration::young()->_active_type = ZYoungType::none;
+}
+
+ZGenerationYoung::ZGenerationYoung(ZPageTable* page_table,
+                                   const ZForwardingTable* old_forwarding_table,
+                                   ZPageAllocator* page_allocator) :
+    ZGeneration(ZGenerationId::young, page_table, page_allocator),
+    _active_type(ZYoungType::none),
+    _tenuring_threshold(0),
+    _remembered(page_table, old_forwarding_table, page_allocator),
+    _jfr_tracer() {
+  ZGeneration::_young = this;
+}
+
+uint ZGenerationYoung::tenuring_threshold() {
+  return _tenuring_threshold;
+}
+
+class ZGenerationCollectionScopeYoung : public StackObj {
+private:
+  ZYoungTypeSetter _type_setter;
+  ZStatTimer       _stat_timer;
+
+public:
+  ZGenerationCollectionScopeYoung(ZYoungType type, ConcurrentGCTimer* gc_timer) :
+      _type_setter(type),
+      _stat_timer(ZPhaseGenerationYoung[(int)type], gc_timer) {
+    // Update statistics and set the GC timer
+    ZGeneration::young()->at_collection_start(gc_timer);
+  }
+
+  ~ZGenerationCollectionScopeYoung() {
+    // Update statistics and clear the GC timer
+    ZGeneration::young()->at_collection_end();
+  }
+};
+
+bool ZGenerationYoung::should_record_stats() {
+  return type() == ZYoungType::minor ||
+         type() == ZYoungType::major_partial_roots;
+}
+
+void ZGenerationYoung::collect(ZYoungType type, ConcurrentGCTimer* timer) {
+  ZGenerationCollectionScopeYoung scope(type, timer);
+
+  // Phase 1: Pause Mark Start
+  pause_mark_start();
+
+  // Phase 2: Concurrent Mark
+  concurrent_mark();
+
+  abortpoint();
+
+  // Phase 3: Pause Mark End
+  while (!pause_mark_end()) {
+    // Phase 3.5: Concurrent Mark Continue
+    concurrent_mark_continue();
+
+    abortpoint();
+  }
+
+  // Phase 4: Concurrent Mark Free
+  concurrent_mark_free();
+
+  abortpoint();
+
+  // Phase 5: Concurrent Reset Relocation Set
+  concurrent_reset_relocation_set();
+
+  abortpoint();
+
+  // Phase 6: Concurrent Select Relocation Set
+  concurrent_select_relocation_set();
+
+  abortpoint();
+
+  // Phase 7: Pause Relocate Start
+  pause_relocate_start();
+
+  // Note that we can't have an abortpoint here. We need
+  // to let concurrent_relocate() call abort_page()
+  // on the remaining entries in the relocation set.
+
+  // Phase 8: Concurrent Relocate
+  concurrent_relocate();
+}
+
+class VM_ZMarkStartYoungAndOld : public VM_ZOperation {
+public:
+  virtual VMOp_Type type() const {
+    return VMOp_ZMarkStartYoungAndOld;
+  }
+
+  virtual bool block_jni_critical() const {
+    return true;
+  }
+
+  virtual bool do_operation() {
+    ZStatTimerYoung timer(ZPhasePauseMarkStartYoungAndOld);
+    ZServiceabilityPauseTracer tracer;
+
+    ZCollectedHeap::heap()->increment_total_collections(true /* full */);
+    ZGeneration::young()->mark_start();
+    ZGeneration::old()->mark_start();
+
+    return true;
+  }
+};
+
+class VM_ZMarkStartYoung : public VM_ZOperation {
+public:
+  virtual VMOp_Type type() const {
+    return VMOp_ZMarkStartYoung;
+  }
+
+  virtual bool block_jni_critical() const {
+    return true;
+  }
+
+  virtual bool do_operation() {
+    ZStatTimerYoung timer(ZPhasePauseMarkStartYoung);
+    ZServiceabilityPauseTracer tracer;
+
+    ZCollectedHeap::heap()->increment_total_collections(false /* full */);
+    ZGeneration::young()->mark_start();
+
+    return true;
+  }
+};
+
+void ZGenerationYoung::flip_mark_start() {
+  ZGlobalsPointers::flip_young_mark_start();
+  ZBarrierSet::assembler()->patch_barriers();
+  ZVerify::on_color_flip();
+}
+
+void ZGenerationYoung::flip_relocate_start() {
+  ZGlobalsPointers::flip_young_relocate_start();
+  ZBarrierSet::assembler()->patch_barriers();
+  ZVerify::on_color_flip();
+}
+
+void ZGenerationYoung::pause_mark_start() {
+  if (type() == ZYoungType::major_full_roots ||
+      type() == ZYoungType::major_partial_roots) {
+    VM_ZMarkStartYoungAndOld().pause();
+  } else {
+    VM_ZMarkStartYoung().pause();
+  }
+}
+
+void ZGenerationYoung::concurrent_mark() {
+  ZStatTimerYoung timer(ZPhaseConcurrentMarkYoung);
+  mark_roots();
+  mark_follow();
+}
+
+class VM_ZMarkEndYoung : public VM_ZOperation {
+public:
+  virtual VMOp_Type type() const {
+    return VMOp_ZMarkEndYoung;
+  }
+
+  virtual bool do_operation() {
+    ZStatTimerYoung timer(ZPhasePauseMarkEndYoung);
+    ZServiceabilityPauseTracer tracer;
+
+    return ZGeneration::young()->mark_end();
+  }
+};
+
+
+bool ZGenerationYoung::pause_mark_end() {
+  return VM_ZMarkEndYoung().pause();
+}
+
+void ZGenerationYoung::concurrent_mark_continue() {
+  ZStatTimerYoung timer(ZPhaseConcurrentMarkContinueYoung);
+  mark_follow();
+}
+
+void ZGenerationYoung::concurrent_mark_free() {
+  ZStatTimerYoung timer(ZPhaseConcurrentMarkFreeYoung);
+  mark_free();
+}
+
+void ZGenerationYoung::concurrent_reset_relocation_set() {
+  ZStatTimerYoung timer(ZPhaseConcurrentResetRelocationSetYoung);
+  reset_relocation_set();
+}
+
+void ZGenerationYoung::select_tenuring_threshold(ZRelocationSetSelectorStats stats, bool promote_all) {
+  const char* reason = "";
+  if (promote_all) {
+    _tenuring_threshold = 0;
+    reason = "Promote All";
+  } else if (ZTenuringThreshold != -1) {
+    _tenuring_threshold = static_cast<uint>(ZTenuringThreshold);
+    reason = "ZTenuringThreshold";
+  } else {
+    _tenuring_threshold = compute_tenuring_threshold(stats);
+    reason = "Computed";
+  }
+  log_info(gc, reloc)("Using tenuring threshold: %d (%s)", _tenuring_threshold, reason);
+}
+
+uint ZGenerationYoung::compute_tenuring_threshold(ZRelocationSetSelectorStats stats) {
+  size_t young_live_total = 0;
+  size_t young_live_last = 0;
+  double young_life_expectancy_sum = 0.0;
+  uint young_life_expectancy_samples = 0;
+  uint last_populated_age = 0;
+  size_t last_populated_live = 0;
+
+  for (uint i = 0; i <= ZPageAgeMax; ++i) {
+    const ZPageAge age = static_cast<ZPageAge>(i);
+    const size_t young_live = stats.small(age).live() + stats.medium(age).live() + stats.large(age).live();
+    if (young_live > 0) {
+      last_populated_age = i;
+      last_populated_live = young_live;
+      if (young_live_last > 0) {
+        young_life_expectancy_sum += double(young_live) / double(young_live_last);
+        young_life_expectancy_samples++;
+      }
+    }
+    young_live_total += young_live;
+    young_live_last = young_live;
+  }
+
+  if (young_live_total == 0) {
+    return 0;
+  }
+
+  const size_t young_used_at_mark_start = ZGeneration::young()->stat_heap()->used_generation_at_mark_start();
+  const size_t young_garbage = ZGeneration::young()->stat_heap()->garbage_at_mark_end();
+  const size_t young_allocated = ZGeneration::young()->stat_heap()->allocated_at_mark_end();
+  const size_t soft_max_capacity = ZHeap::heap()->soft_max_capacity();
+
+  // The life expectancy shows by what factor on average one age changes between
+  // two ages in the age table. Values below 1 indicate generational behaviour where
+  // the live bytes is shrinking from age to age. Values at or above 1 indicate
+  // anti-generational patterns where the live bytes isn't going down or grows
+  // from age to age.
+  const double young_life_expectancy = young_life_expectancy_samples == 0 ? 1.0 : young_life_expectancy_sum / young_life_expectancy_samples;
+
+  // The life decay factor is the reciprocal of the life expectancy. Therefore,
+  // values at or below 1 indicate anti-generational behaviour where the live
+  // bytes either stays the same or grows from age to age. Conversely, values
+  // above 1 indicate generational behaviour where the live bytes shrinks from
+  // age to age. The more it shrinks from age to age, the higher the value.
+  // Therefore, the higher this value is, the higher we want the tenuring
+  // threshold to be, as we exponentially avoid promotions to the old generation.
+  const double young_life_decay_factor = 1.0 / young_life_expectancy;
+
+  // The young residency reciprocal indicates the inverse of how small the
+  // resident part of the young generation is compared to the entire heap. Values
+  // below 1 indicate it is relatively big. Conversely, values above 1 indicate
+  // it is relatively small.
+  const double young_residency_reciprocal = double(soft_max_capacity) / double(young_live_total);
+
+  // The old residency factor clamps the old residency reciprocal to
+  // at least 1. That implies this factor is 1 unless the resident memory of
+  // the old generation is small compared to the residency of the heap. The
+  // smaller the old generation is, the higher this value is. The reasoning
+  // is that the less memory that is resident in the old generation, the less
+  // point there is in promoting objects to the old generation, as the amount
+  // of work it removes from the young generation collections becomes less
+  // and less valuable, the smaller the old generation is.
+  const double young_residency_factor = MAX2(young_residency_reciprocal, 1.0);
+
+  // The allocated to garbage ratio, compares the ratio of newly allocated
+  // memory since GC started to how much garbage we are freeing up. The higher
+  // the value, the harder it is for the YC to keep up with the allocation rate.
+  const double allocated_garbage_ratio = double(young_allocated) / double(young_garbage + 1);
+
+  // We slow down the young residency factor with a log. A larger log slows
+  // it down faster. We select a log between 2 - 16 scaled by the allocated
+  // to garbage factor. This selects a larger log when the GC has a harder
+  // time keeping up, which causes more promotions to the old generation,
+  // making the young collections faster so they can catch up.
+  const double young_log = MAX2(MIN2(allocated_garbage_ratio, 1.0) * 16, 2.0);
+
+  // The young log residency is essentially the young residency factor, but slowed
+  // down by the log_{young_log}(X) function described above.
+  const double young_log_residency = log(young_residency_factor) / log(young_log);
+
+  // The tenuring threshold is computed as the young life decay factor times
+  // the young residency factor. That takes into consideration that the
+  // value should be higher the more generational the age table is, and higher
+  // the more insignificant the footprint of young resident memory is, yet breaks
+  // if the GC is finding it hard to keep up with the allocation rate.
+  const double tenuring_threshold_raw = young_life_decay_factor * young_log_residency;
+
+  log_trace(gc, reloc)("Young Allocated: " SIZE_FORMAT "M", young_allocated / M);
+  log_trace(gc, reloc)("Young Garbage: " SIZE_FORMAT "M", young_garbage / M);
+  log_debug(gc, reloc)("Allocated To Garbage: %.1f", allocated_garbage_ratio);
+  log_trace(gc, reloc)("Young Log: %.1f", young_log);
+  log_trace(gc, reloc)("Young Residency Reciprocal: %.1f", young_residency_reciprocal);
+  log_trace(gc, reloc)("Young Residency Factor: %.1f", young_residency_factor);
+  log_debug(gc, reloc)("Young Log Residency: %.1f", young_log_residency);
+  log_debug(gc, reloc)("Life Decay Factor: %.1f", young_life_decay_factor);
+
+  // Round to an integer as we can't have non-integral tenuring threshold.
+  const uint upper_bound = MIN2(last_populated_age + 1u, (uint)MaxTenuringThreshold);
+  const uint lower_bound = MIN2(1u, upper_bound);
+  const uint tenuring_threshold = clamp((uint)round(tenuring_threshold_raw), lower_bound, upper_bound);
+
+  return tenuring_threshold;
+}
+
+void ZGenerationYoung::concurrent_select_relocation_set() {
+  ZStatTimerYoung timer(ZPhaseConcurrentSelectRelocationSetYoung);
+  const bool promote_all = type() == ZYoungType::major_full_preclean;
+  select_relocation_set(_id, promote_all);
+}
+
+class VM_ZRelocateStartYoung : public VM_ZOperation {
+public:
+  virtual VMOp_Type type() const {
+    return VMOp_ZRelocateStartYoung;
+  }
+
+  virtual bool block_jni_critical() const {
+    return true;
+  }
+
+  virtual bool do_operation() {
+    ZStatTimerYoung timer(ZPhasePauseRelocateStartYoung);
+    ZServiceabilityPauseTracer tracer;
+
+    ZGeneration::young()->relocate_start();
+
+    return true;
+  }
+};
+
+void ZGenerationYoung::pause_relocate_start() {
+  VM_ZRelocateStartYoung().pause();
+}
+
+void ZGenerationYoung::concurrent_relocate() {
+  ZStatTimerYoung timer(ZPhaseConcurrentRelocatedYoung);
+  relocate();
+}
+
+void ZGenerationYoung::mark_start() {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+
+  // Change good colors
+  flip_mark_start();
+
+  // Retire allocating pages
+  ZAllocator::eden()->retire_pages();
+  for (ZPageAge i = ZPageAge::survivor1; i <= ZPageAge::survivor14; i = static_cast<ZPageAge>(static_cast<uint>(i) + 1)) {
+    ZAllocator::relocation(i)->retire_pages();
+  }
+
+  // Reset allocated/reclaimed/used statistics
+  reset_statistics();
+
+  // Increment sequence number
+  _seqnum++;
+
+  // Enter mark phase
+  set_phase(Phase::Mark);
+
+  // Reset marking information and mark roots
+  _mark.start();
+
+  // Flip remembered set bits
+  _remembered.flip();
+
+  // Update statistics
+  stat_heap()->at_mark_start(_page_allocator->stats(this));
+}
+
+void ZGenerationYoung::mark_roots() {
+  ZStatTimerYoung timer(ZSubPhaseConcurrentMarkRootsYoung);
+  _mark.mark_young_roots();
+}
+
+void ZGenerationYoung::mark_follow() {
+  // Combine following with scanning the remembered set
+  ZStatTimerYoung timer(ZSubPhaseConcurrentMarkFollowYoung);
+  _remembered.scan_and_follow(&_mark);
+}
+
+bool ZGenerationYoung::mark_end() {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+
+  // End marking
+  if (!_mark.end()) {
+    // Marking not completed, continue concurrent mark
+    return false;
+  }
+
+  // Enter mark completed phase
+  set_phase(Phase::MarkComplete);
+
+  // Update statistics
+  stat_heap()->at_mark_end(_page_allocator->stats(this));
+
+  // Notify JVMTI that some tagmap entry objects may have died.
+  JvmtiTagMap::set_needs_cleaning();
+
+  return true;
+}
+
+void ZGenerationYoung::relocate_start() {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+
+  // Change good colors
+  flip_relocate_start();
+
+  // Enter relocate phase
+  set_phase(Phase::Relocate);
+
+  // Update statistics
+  stat_heap()->at_relocate_start(_page_allocator->stats(this));
+
+  _relocate.start();
+}
+
+void ZGenerationYoung::relocate() {
+  // Relocate relocation set
+  _relocate.relocate(&_relocation_set);
+
+  // Update statistics
+  stat_heap()->at_relocate_end(_page_allocator->stats(this), should_record_stats());
+}
+
+void ZGenerationYoung::flip_promote(ZPage* from_page, ZPage* to_page) {
+  _page_table->replace(from_page, to_page);
+
+  // Update statistics
+  _page_allocator->promote_used(from_page->size());
+  increase_freed(from_page->size());
+  increase_promoted(from_page->live_bytes());
+}
+
+void ZGenerationYoung::in_place_relocate_promote(ZPage* from_page, ZPage* to_page) {
+  _page_table->replace(from_page, to_page);
+
+  // Update statistics
+  _page_allocator->promote_used(from_page->size());
+}
+
+void ZGenerationYoung::register_flip_promoted(const ZArray<ZPage*>& pages) {
+  _relocation_set.register_flip_promoted(pages);
+}
+
+void ZGenerationYoung::register_in_place_relocate_promoted(ZPage* page) {
+  _relocation_set.register_in_place_relocate_promoted(page);
+}
+
+void ZGenerationYoung::register_with_remset(ZPage* page) {
+  _remembered.register_found_old(page);
+}
+
+ZGenerationTracer* ZGenerationYoung::jfr_tracer() {
+  return &_jfr_tracer;
+}
+
+ZGenerationOld::ZGenerationOld(ZPageTable* page_table, ZPageAllocator* page_allocator) :
+    ZGeneration(ZGenerationId::old, page_table, page_allocator),
+    _reference_processor(&_workers),
+    _weak_roots_processor(&_workers),
+    _unload(&_workers),
+    _total_collections_at_start(0),
+    _young_seqnum_at_reloc_start(0),
+    _jfr_tracer() {
+  ZGeneration::_old = this;
+}
+
+class ZGenerationCollectionScopeOld : public StackObj {
+private:
+  ZStatTimer      _stat_timer;
+  ZDriverUnlocker _unlocker;
+
+public:
+  ZGenerationCollectionScopeOld(ConcurrentGCTimer* gc_timer) :
+      _stat_timer(ZPhaseGenerationOld, gc_timer),
+      _unlocker() {
+    // Update statistics and set the GC timer
+    ZGeneration::old()->at_collection_start(gc_timer);
+  }
+
+  ~ZGenerationCollectionScopeOld() {
+    // Update statistics and clear the GC timer
+    ZGeneration::old()->at_collection_end();
+  }
+};
+
+bool ZGenerationOld::should_record_stats() {
+  return true;
+}
+
+void ZGenerationOld::collect(ConcurrentGCTimer* timer) {
+  ZGenerationCollectionScopeOld scope(timer);
+
+  // Phase 1: Concurrent Mark
+  concurrent_mark();
+
+  abortpoint();
+
+  // Phase 2: Pause Mark End
+  while (!pause_mark_end()) {
+    // Phase 2.5: Concurrent Mark Continue
+    concurrent_mark_continue();
+
+    abortpoint();
+  }
+
+  // Phase 3: Concurrent Mark Free
+  concurrent_mark_free();
+
+  abortpoint();
+
+  // Phase 4: Concurrent Process Non-Strong References
+  concurrent_process_non_strong_references();
+
+  abortpoint();
+
+  // Phase 5: Concurrent Reset Relocation Set
+  concurrent_reset_relocation_set();
+
+  abortpoint();
+
+  // Phase 6: Pause Verify
+  pause_verify();
+
+  // Phase 7: Concurrent Select Relocation Set
+  concurrent_select_relocation_set();
+
+  abortpoint();
+
+  {
+    ZDriverLocker locker;
+
+    // Phase 8: Concurrent Remap Roots
+    concurrent_remap_young_roots();
+
+    abortpoint();
+
+    // Phase 9: Pause Relocate Start
+    pause_relocate_start();
+  }
+
+  // Note that we can't have an abortpoint here. We need
+  // to let concurrent_relocate() call abort_page()
+  // on the remaining entries in the relocation set.
+
+  // Phase 10: Concurrent Relocate
+  concurrent_relocate();
+}
+
+void ZGenerationOld::flip_mark_start() {
+  ZGlobalsPointers::flip_old_mark_start();
+  ZBarrierSet::assembler()->patch_barriers();
+  ZVerify::on_color_flip();
+}
+
+void ZGenerationOld::flip_relocate_start() {
+  ZGlobalsPointers::flip_old_relocate_start();
+  ZBarrierSet::assembler()->patch_barriers();
+  ZVerify::on_color_flip();
+}
+
+void ZGenerationOld::concurrent_mark() {
+  ZStatTimerOld timer(ZPhaseConcurrentMarkOld);
+  ZBreakpoint::at_after_marking_started();
+  mark_roots();
+  mark_follow();
+  ZBreakpoint::at_before_marking_completed();
+}
+
+class VM_ZMarkEndOld : public VM_ZOperation {
+public:
+  virtual VMOp_Type type() const {
+    return VMOp_ZMarkEndOld;
+  }
+
+  virtual bool do_operation() {
+    ZStatTimerOld timer(ZPhasePauseMarkEndOld);
+    ZServiceabilityPauseTracer tracer;
+
+    return ZGeneration::old()->mark_end();
+  }
+};
+
+bool ZGenerationOld::pause_mark_end() {
+  return VM_ZMarkEndOld().pause();
+}
+
+void ZGenerationOld::concurrent_mark_continue() {
+  ZStatTimerOld timer(ZPhaseConcurrentMarkContinueOld);
+  mark_follow();
+}
+
+void ZGenerationOld::concurrent_mark_free() {
+  ZStatTimerOld timer(ZPhaseConcurrentMarkFreeOld);
+  mark_free();
+}
+
+void ZGenerationOld::concurrent_process_non_strong_references() {
+  ZStatTimerOld timer(ZPhaseConcurrentProcessNonStrongOld);
+  ZBreakpoint::at_after_reference_processing_started();
+  process_non_strong_references();
+}
+
+void ZGenerationOld::concurrent_reset_relocation_set() {
+  ZStatTimerOld timer(ZPhaseConcurrentResetRelocationSetOld);
+  reset_relocation_set();
+}
+
+class VM_ZVerifyOld : public VM_Operation {
+public:
+  virtual VMOp_Type type() const {
+    return VMOp_ZVerifyOld;
+  }
+
+  virtual bool skip_thread_oop_barriers() const {
+    return true;
+  }
+
+  virtual void doit() {
+    ZVerify::after_weak_processing();
+  }
+
+  void pause() {
+    VMThread::execute(this);
+  }
+};
+
+void ZGenerationOld::pause_verify() {
+  // Note that we block out concurrent young collections when performing the
+  // verification. The verification checks that store good oops in the
+  // old generation have a corresponding remembered set entry, or is in
+  // a store barrier buffer (hence asynchronously creating such entries).
+  // That lookup would otherwise race with installation of base pointers
+  // into the store barrier buffer. We dodge that race by blocking out
+  // young collections during this verification.
+  if (ZVerifyRoots || ZVerifyObjects) {
+    // Limited verification
+    ZDriverLocker locker;
+    VM_ZVerifyOld().pause();
+  }
+}
+
+void ZGenerationOld::concurrent_select_relocation_set() {
+  ZStatTimerOld timer(ZPhaseConcurrentSelectRelocationSetOld);
+  select_relocation_set(_id, false /* promote_all */);
+}
+
+class VM_ZRelocateStartOld : public VM_ZOperation {
+public:
+  virtual VMOp_Type type() const {
+    return VMOp_ZRelocateStartOld;
+  }
+
+  virtual bool block_jni_critical() const {
+    return true;
+  }
+
+  virtual bool do_operation() {
+    ZStatTimerOld timer(ZPhasePauseRelocateStartOld);
+    ZServiceabilityPauseTracer tracer;
+
+    ZGeneration::old()->relocate_start();
+
+    return true;
+  }
+};
+
+void ZGenerationOld::pause_relocate_start() {
+  VM_ZRelocateStartOld().pause();
+}
+
+void ZGenerationOld::concurrent_relocate() {
+  ZStatTimerOld timer(ZPhaseConcurrentRelocatedOld);
+  relocate();
+}
+
+void ZGenerationOld::concurrent_remap_young_roots() {
+  ZStatTimerOld timer(ZPhaseConcurrentRemapRootsOld);
+  remap_young_roots();
+}
+
+void ZGenerationOld::mark_start() {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+
+  // Verification
+  ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_strong);
+
+  // Change good colors
+  flip_mark_start();
+
+  // Retire allocating pages
+  ZAllocator::old()->retire_pages();
+
+  // Reset allocated/reclaimed/used statistics
+  reset_statistics();
+
+  // Reset encountered/dropped/enqueued statistics
+  _reference_processor.reset_statistics();
+
+  // Increment sequence number
+  _seqnum++;
+
+  // Enter mark phase
+  set_phase(Phase::Mark);
+
+  // Reset marking information and mark roots
+  _mark.start();
+
+  // Update statistics
+  stat_heap()->at_mark_start(_page_allocator->stats(this));
+
+  // Note that we start a marking cycle.
+  // Unlike other GCs, the color switch implicitly changes the nmethods
+  // to be armed, and the thread-local disarm values are lazily updated
+  // when JavaThreads wake up from safepoints.
+  CodeCache::on_gc_marking_cycle_start();
+
+  _total_collections_at_start = ZCollectedHeap::heap()->total_collections();
+}
+
+void ZGenerationOld::mark_roots() {
+  ZStatTimerOld timer(ZSubPhaseConcurrentMarkRootsOld);
+  _mark.mark_old_roots();
+}
+
+void ZGenerationOld::mark_follow() {
+  ZStatTimerOld timer(ZSubPhaseConcurrentMarkFollowOld);
+  _mark.mark_follow();
+}
+
+bool ZGenerationOld::mark_end() {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+
+  // Try end marking
+  if (!_mark.end()) {
+    // Marking not completed, continue concurrent mark
+    return false;
+  }
+
+  // Enter mark completed phase
+  set_phase(Phase::MarkComplete);
+
+  // Verify after mark
+  ZVerify::after_mark();
+
+  // Update statistics
+  stat_heap()->at_mark_end(_page_allocator->stats(this));
+
+  // Block resurrection of weak/phantom references
+  ZResurrection::block();
+
+  // Prepare to unload stale metadata and nmethods
+  _unload.prepare();
+
+  // Notify JVMTI that some tagmap entry objects may have died.
+  JvmtiTagMap::set_needs_cleaning();
+
+  // Note that we finished a marking cycle.
+  // Unlike other GCs, we do not arm the nmethods
+  // when marking terminates.
+  CodeCache::on_gc_marking_cycle_finish();
+
+  return true;
+}
+
+void ZGenerationOld::set_soft_reference_policy(bool clear) {
+  _reference_processor.set_soft_reference_policy(clear);
+}
+
+class ZRendezvousHandshakeClosure : public HandshakeClosure {
+public:
+  ZRendezvousHandshakeClosure() :
+      HandshakeClosure("ZRendezvous") {}
+
+  void do_thread(Thread* thread) {
+    // Does nothing
+  }
+};
+
+class ZRendezvousGCThreads: public VM_Operation {
+ public:
+  VMOp_Type type() const { return VMOp_ZRendezvousGCThreads; }
+
+  virtual bool evaluate_at_safepoint() const {
+    // We only care about synchronizing the GC threads.
+    // Leave the Java threads running.
+    return false;
+  }
+
+  virtual bool skip_thread_oop_barriers() const {
+    fatal("Concurrent VMOps should not call this");
+    return true;
+  }
+
+  void doit() {
+    // Light weight "handshake" of the GC threads
+    SuspendibleThreadSet::synchronize();
+    SuspendibleThreadSet::desynchronize();
+  };
+};
+
+
+void ZGenerationOld::process_non_strong_references() {
+  // Process Soft/Weak/Final/PhantomReferences
+  _reference_processor.process_references();
+
+  // Process weak roots
+  _weak_roots_processor.process_weak_roots();
+
+  // Unlink stale metadata and nmethods
+  _unload.unlink();
+
+  // Perform a handshake. This is needed 1) to make sure that stale
+  // metadata and nmethods are no longer observable. And 2), to
+  // prevent the race where a mutator first loads an oop, which is
+  // logically null but not yet cleared. Then this oop gets cleared
+  // by the reference processor and resurrection is unblocked. At
+  // this point the mutator could see the unblocked state and pass
+  // this invalid oop through the normal barrier path, which would
+  // incorrectly try to mark the oop.
+  ZRendezvousHandshakeClosure cl;
+  Handshake::execute(&cl);
+
+  // GC threads are not part of the handshake above.
+  // Explicitly "handshake" them.
+  ZRendezvousGCThreads op;
+  VMThread::execute(&op);
+
+  // Unblock resurrection of weak/phantom references
+  ZResurrection::unblock();
+
+  // Purge stale metadata and nmethods that were unlinked
+  _unload.purge();
+
+  // Enqueue Soft/Weak/Final/PhantomReferences. Note that this
+  // must be done after unblocking resurrection. Otherwise the
+  // Finalizer thread could call Reference.get() on the Finalizers
+  // that were just enqueued, which would incorrectly return null
+  // during the resurrection block window, since such referents
+  // are only Finalizable marked.
+  _reference_processor.enqueue_references();
+
+  // Clear old markings claim bits.
+  // Note: Clearing _claim_strong also clears _claim_finalizable.
+  ClassLoaderDataGraph::clear_claimed_marks(ClassLoaderData::_claim_strong);
+}
+
+void ZGenerationOld::relocate_start() {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+
+  // Finish unloading stale metadata and nmethods
+  _unload.finish();
+
+  // Change good colors
+  flip_relocate_start();
+
+  // Enter relocate phase
+  set_phase(Phase::Relocate);
+
+  // Update statistics
+  stat_heap()->at_relocate_start(_page_allocator->stats(this));
+
+  // Need to know the remset parity when relocating objects
+  _young_seqnum_at_reloc_start = ZGeneration::young()->seqnum();
+
+  _relocate.start();
+}
+
+void ZGenerationOld::relocate() {
+  // Relocate relocation set
+  _relocate.relocate(&_relocation_set);
+
+  // Update statistics
+  stat_heap()->at_relocate_end(_page_allocator->stats(this), should_record_stats());
+}
+
+class ZRemapOopClosure : public OopClosure {
+public:
+  virtual void do_oop(oop* p) {
+    ZBarrier::load_barrier_on_oop_field((volatile zpointer*)p);
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+};
+
+class ZRemapThreadClosure : public ThreadClosure {
+public:
+  virtual void do_thread(Thread* thread) {
+    JavaThread* const jt = JavaThread::cast(thread);
+    StackWatermarkSet::finish_processing(jt, nullptr, StackWatermarkKind::gc);
+  }
+};
+
+class ZRemapNMethodClosure : public NMethodClosure {
+private:
+  ZBarrierSetNMethod* const _bs_nm;
+
+public:
+  ZRemapNMethodClosure() :
+      _bs_nm(static_cast<ZBarrierSetNMethod*>(BarrierSet::barrier_set()->barrier_set_nmethod())) {}
+
+  virtual void do_nmethod(nmethod* nm) {
+    ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
+    if (_bs_nm->is_armed(nm)) {
+      // Heal barriers
+      ZNMethod::nmethod_patch_barriers(nm);
+
+      // Heal oops
+      ZUncoloredRootProcessOopClosure cl(ZNMethod::color(nm));
+      ZNMethod::nmethod_oops_do_inner(nm, &cl);
+
+      log_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by old remapping", p2i(nm));
+
+      // Disarm
+      _bs_nm->disarm(nm);
+    }
+  }
+};
+
+typedef ClaimingCLDToOopClosure<ClassLoaderData::_claim_none> ZRemapCLDClosure;
+
+class ZRemapYoungRootsTask : public ZTask {
+private:
+  ZGenerationPagesParallelIterator _old_pages_parallel_iterator;
+
+  ZRootsIteratorAllColored         _roots_colored;
+  ZRootsIteratorAllUncolored       _roots_uncolored;
+
+  ZRemapOopClosure                 _cl_colored;
+  ZRemapCLDClosure                 _cld_cl;
+
+  ZRemapThreadClosure              _thread_cl;
+  ZRemapNMethodClosure             _nm_cl;
+
+public:
+  ZRemapYoungRootsTask(ZPageTable* page_table, ZPageAllocator* page_allocator) :
+      ZTask("ZRemapYoungRootsTask"),
+      _old_pages_parallel_iterator(page_table, ZGenerationId::old, page_allocator),
+      _roots_colored(ZGenerationIdOptional::old),
+      _roots_uncolored(ZGenerationIdOptional::old),
+      _cl_colored(),
+      _cld_cl(&_cl_colored),
+      _thread_cl(),
+      _nm_cl() {
+    ClassLoaderDataGraph_lock->lock();
+  }
+
+  ~ZRemapYoungRootsTask() {
+    ClassLoaderDataGraph_lock->unlock();
+  }
+
+  virtual void work() {
+    {
+      ZStatTimerWorker timer(ZSubPhaseConcurrentRemapRootsColoredOld);
+      _roots_colored.apply(&_cl_colored,
+                           &_cld_cl);
+    }
+
+    {
+      ZStatTimerWorker timer(ZSubPhaseConcurrentRemapRootsUncoloredOld);
+      _roots_uncolored.apply(&_thread_cl,
+                             &_nm_cl);
+    }
+
+    {
+      ZStatTimerWorker timer(ZSubPhaseConcurrentRemapRememberedOld);
+      _old_pages_parallel_iterator.do_pages([&](ZPage* page) {
+        // Visit all object fields that potentially pointing into young generation
+        page->oops_do_current_remembered(ZBarrier::load_barrier_on_oop_field);
+        return true;
+      });
+    }
+  }
+};
+
+// This function is used by the old generation to purge roots to the young generation from
+// young remap bit errors, before the old generation performs old relocate start. By doing
+// that, we can know that double remap bit errors don't need to be concerned with double
+// remap bit errors, in the young generation roots. That makes it possible to figure out
+// which generation table to use when remapping a pointer, without needing an extra adjust
+// phase that walks the entire heap.
+void ZGenerationOld::remap_young_roots() {
+  // We upgrade the number of workers to the number last used by the young generation. The
+  // reason is that this code is run under the driver lock, which means that a young generation
+  // collection might be waiting for this code to complete.
+  uint prev_nworkers = _workers.active_workers();
+  uint remap_nworkers = clamp(ZGeneration::young()->workers()->active_workers() + prev_nworkers, 1u, ZOldGCThreads);
+  _workers.set_active_workers(remap_nworkers);
+
+  // TODO: The STS joiner is only needed to satisfy z_assert_is_barrier_safe that doesn't
+  // understand the driver locker. Consider making the assert aware of the driver locker.
+  SuspendibleThreadSetJoiner sts_joiner;
+
+  ZRemapYoungRootsTask task(_page_table, _page_allocator);
+  workers()->run(&task);
+  _workers.set_active_workers(prev_nworkers);
+}
+
+uint ZGenerationOld::total_collections_at_start() const {
+  return _total_collections_at_start;
+}
+
+ZGenerationTracer* ZGenerationOld::jfr_tracer() {
+  return &_jfr_tracer;
+}
diff --git a/src/hotspot/share/gc/z/zGeneration.hpp b/src/hotspot/share/gc/z/zGeneration.hpp
new file mode 100644
index 00000000000..23736f45b7b
--- /dev/null
+++ b/src/hotspot/share/gc/z/zGeneration.hpp
@@ -0,0 +1,322 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZGENERATION_HPP
+#define SHARE_GC_Z_ZGENERATION_HPP
+
+#include "gc/z/zForwardingTable.hpp"
+#include "gc/z/zGenerationId.hpp"
+#include "gc/z/zMark.hpp"
+#include "gc/z/zReferenceProcessor.hpp"
+#include "gc/z/zRelocate.hpp"
+#include "gc/z/zRelocationSet.hpp"
+#include "gc/z/zRemembered.hpp"
+#include "gc/z/zStat.hpp"
+#include "gc/z/zTracer.hpp"
+#include "gc/z/zUnload.hpp"
+#include "gc/z/zWeakRootsProcessor.hpp"
+#include "gc/z/zWorkers.hpp"
+#include "memory/allocation.hpp"
+
+class ThreadClosure;
+class ZForwardingTable;
+class ZGenerationOld;
+class ZGenerationYoung;
+class ZPage;
+class ZPageAllocator;
+class ZPageTable;
+class ZRelocationSetSelector;
+
+class ZGeneration {
+  friend class ZForwardingTest;
+  friend class ZLiveMapTest;
+
+protected:
+  static ZGenerationYoung* _young;
+  static ZGenerationOld*   _old;
+
+  enum class Phase {
+    Mark,
+    MarkComplete,
+    Relocate
+  };
+
+  const ZGenerationId   _id;
+  ZPageAllocator* const _page_allocator;
+  ZPageTable* const     _page_table;
+  ZForwardingTable      _forwarding_table;
+  ZWorkers              _workers;
+  ZMark                 _mark;
+  ZRelocate             _relocate;
+  ZRelocationSet        _relocation_set;
+
+  volatile size_t       _freed;
+  volatile size_t       _promoted;
+  volatile size_t       _compacted;
+
+  Phase                 _phase;
+  uint32_t              _seqnum;
+
+  ZStatHeap             _stat_heap;
+  ZStatCycle            _stat_cycle;
+  ZStatWorkers          _stat_workers;
+  ZStatMark             _stat_mark;
+  ZStatRelocation       _stat_relocation;
+
+  ConcurrentGCTimer*    _gc_timer;
+
+  void free_empty_pages(ZRelocationSetSelector* selector, int bulk);
+  void flip_age_pages(const ZRelocationSetSelector* selector);
+  void flip_age_pages(const ZArray<ZPage*>* pages);
+
+  void mark_free();
+
+  void select_relocation_set(ZGenerationId generation, bool promote_all);
+  void reset_relocation_set();
+
+  ZGeneration(ZGenerationId id, ZPageTable* page_table, ZPageAllocator* page_allocator);
+
+  void log_phase_switch(Phase from, Phase to);
+
+public:
+  bool is_initialized() const;
+
+  // GC phases
+  void set_phase(Phase new_phase);
+  bool is_phase_relocate() const;
+  bool is_phase_mark() const;
+  bool is_phase_mark_complete() const;
+  const char* phase_to_string() const;
+
+  uint32_t seqnum() const;
+
+  ZGenerationId id() const;
+  ZGenerationIdOptional id_optional() const;
+  bool is_young() const;
+  bool is_old() const;
+
+  static ZGenerationYoung* young();
+  static ZGenerationOld* old();
+  static ZGeneration* generation(ZGenerationId id);
+
+  // Statistics
+  void reset_statistics();
+  virtual bool should_record_stats() = 0;
+  ssize_t freed() const;
+  void increase_freed(size_t size);
+  size_t promoted() const;
+  void increase_promoted(size_t size);
+  size_t compacted() const;
+  void increase_compacted(size_t size);
+
+  ConcurrentGCTimer* gc_timer() const;
+  void set_gc_timer(ConcurrentGCTimer* gc_timer);
+  void clear_gc_timer();
+
+  ZStatHeap* stat_heap();
+  ZStatCycle* stat_cycle();
+  ZStatWorkers* stat_workers();
+  ZStatMark* stat_mark();
+  ZStatRelocation* stat_relocation();
+
+  void at_collection_start(ConcurrentGCTimer* gc_timer);
+  void at_collection_end();
+
+  // Workers
+  ZWorkers* workers();
+  uint active_workers() const;
+  void set_active_workers(uint nworkers);
+
+  // Worker resizing
+  bool should_worker_resize();
+
+  ZPageTable* page_table() const;
+  const ZForwardingTable* forwarding_table() const;
+  ZForwarding* forwarding(zaddress_unsafe addr) const;
+
+  ZRelocationSetParallelIterator relocation_set_parallel_iterator();
+
+  // Marking
+  template <bool resurrect, bool gc_thread, bool follow, bool finalizable>
+  void mark_object(zaddress addr);
+  template <bool resurrect, bool gc_thread, bool follow, bool finalizable>
+  void mark_object_if_active(zaddress addr);
+  void mark_flush_and_free(Thread* thread);
+
+  // Relocation
+  void synchronize_relocation();
+  void desynchronize_relocation();
+  zaddress relocate_or_remap_object(zaddress_unsafe addr);
+  zaddress remap_object(zaddress_unsafe addr);
+
+  // Threads
+  void threads_do(ThreadClosure* tc) const;
+};
+
+enum class ZYoungType {
+  minor,
+  major_full_preclean,
+  major_full_roots,
+  major_partial_roots,
+  none
+};
+
+class ZYoungTypeSetter {
+public:
+  ZYoungTypeSetter(ZYoungType type);
+  ~ZYoungTypeSetter();
+};
+
+class ZGenerationYoung : public ZGeneration {
+  friend class VM_ZMarkEndYoung;
+  friend class VM_ZMarkStartYoung;
+  friend class VM_ZMarkStartYoungAndOld;
+  friend class VM_ZRelocateStartYoung;
+  friend class ZYoungTypeSetter;
+
+private:
+  ZYoungType   _active_type;
+  uint         _tenuring_threshold;
+  ZRemembered  _remembered;
+  ZYoungTracer _jfr_tracer;
+
+  void flip_mark_start();
+  void flip_relocate_start();
+
+  void mark_start();
+  void mark_roots();
+  void mark_follow();
+  bool mark_end();
+  void relocate_start();
+  void relocate();
+
+  void pause_mark_start();
+  void concurrent_mark();
+  bool pause_mark_end();
+  void concurrent_mark_continue();
+  void concurrent_mark_free();
+  void concurrent_reset_relocation_set();
+  void concurrent_select_relocation_set();
+  void pause_relocate_start();
+  void concurrent_relocate();
+
+public:
+  ZGenerationYoung(ZPageTable* page_table,
+                   const ZForwardingTable* old_forwarding_table,
+                   ZPageAllocator* page_allocator);
+
+  ZYoungType type() const;
+
+  void collect(ZYoungType type, ConcurrentGCTimer* timer);
+
+  // Statistics
+  bool should_record_stats();
+
+  // Support for promoting object to the old generation
+  void flip_promote(ZPage* from_page, ZPage* to_page);
+  void in_place_relocate_promote(ZPage* from_page, ZPage* to_page);
+
+  void register_flip_promoted(const ZArray<ZPage*>& pages);
+  void register_in_place_relocate_promoted(ZPage* page);
+
+  uint tenuring_threshold();
+  void select_tenuring_threshold(ZRelocationSetSelectorStats stats, bool promote_all);
+  uint compute_tenuring_threshold(ZRelocationSetSelectorStats stats);
+
+  // Add remembered set entries
+  void remember(volatile zpointer* p);
+  void remember_fields(zaddress addr);
+
+  // Scan a remembered set entry
+  void scan_remembered_field(volatile zpointer* p);
+
+  // Register old pages with remembered set
+  void register_with_remset(ZPage* page);
+
+  // Serviceability
+  ZGenerationTracer* jfr_tracer();
+
+  // Verification
+  bool is_remembered(volatile zpointer* p) const;
+};
+
+class ZGenerationOld : public ZGeneration {
+  friend class VM_ZMarkEndOld;
+  friend class VM_ZMarkStartYoungAndOld;
+  friend class VM_ZRelocateStartOld;
+
+private:
+  ZReferenceProcessor _reference_processor;
+  ZWeakRootsProcessor _weak_roots_processor;
+  ZUnload             _unload;
+  uint                _total_collections_at_start;
+  uint32_t            _young_seqnum_at_reloc_start;
+  ZOldTracer          _jfr_tracer;
+
+  void flip_mark_start();
+  void flip_relocate_start();
+
+  void mark_start();
+  void mark_roots();
+  void mark_follow();
+  bool mark_end();
+  void process_non_strong_references();
+  void relocate_start();
+  void relocate();
+  void remap_young_roots();
+
+  void concurrent_mark();
+  bool pause_mark_end();
+  void concurrent_mark_continue();
+  void concurrent_mark_free();
+  void concurrent_process_non_strong_references();
+  void concurrent_reset_relocation_set();
+  void pause_verify();
+  void concurrent_select_relocation_set();
+  void pause_relocate_start();
+  void concurrent_relocate();
+  void concurrent_remap_young_roots();
+
+public:
+  ZGenerationOld(ZPageTable* page_table, ZPageAllocator* page_allocator);
+
+  void collect(ConcurrentGCTimer* timer);
+
+  // Statistics
+  bool should_record_stats();
+
+  // Reference processing
+  ReferenceDiscoverer* reference_discoverer();
+  void set_soft_reference_policy(bool clear);
+
+  uint total_collections_at_start() const;
+
+  bool active_remset_is_current() const;
+
+  ZRelocateQueue* relocate_queue();
+
+  // Serviceability
+  ZGenerationTracer* jfr_tracer();
+};
+
+#endif // SHARE_GC_Z_ZGENERATION_HPP
diff --git a/src/hotspot/share/gc/z/zGeneration.inline.hpp b/src/hotspot/share/gc/z/zGeneration.inline.hpp
new file mode 100644
index 00000000000..3f138e0acbb
--- /dev/null
+++ b/src/hotspot/share/gc/z/zGeneration.inline.hpp
@@ -0,0 +1,188 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZGENERATION_INLINE_HPP
+#define SHARE_GC_Z_ZGENERATION_INLINE_HPP
+
+#include "gc/z/zGeneration.hpp"
+
+#include "gc/z/zAbort.inline.hpp"
+#include "gc/z/zHeap.inline.hpp"
+#include "gc/z/zWorkers.inline.hpp"
+#include "utilities/debug.hpp"
+
+inline bool ZGeneration::is_phase_relocate() const {
+  return _phase == Phase::Relocate;
+}
+
+inline bool ZGeneration::is_phase_mark() const {
+  return _phase == Phase::Mark;
+}
+
+inline bool ZGeneration::is_phase_mark_complete() const {
+  return _phase == Phase::MarkComplete;
+}
+
+inline uint32_t ZGeneration::seqnum() const {
+  return _seqnum;
+}
+
+inline ZGenerationId ZGeneration::id() const {
+  return _id;
+}
+
+inline ZGenerationIdOptional ZGeneration::id_optional() const {
+  return static_cast<ZGenerationIdOptional>(_id);
+}
+
+inline bool ZGeneration::is_young() const {
+  return _id == ZGenerationId::young;
+}
+
+inline bool ZGeneration::is_old() const {
+  return _id == ZGenerationId::old;
+}
+
+inline ZGenerationYoung* ZGeneration::young() {
+  return _young;
+}
+
+inline ZGenerationOld* ZGeneration::old() {
+  return _old;
+}
+
+inline ZGeneration* ZGeneration::generation(ZGenerationId id) {
+  if (id == ZGenerationId::young) {
+    return _young;
+  } else {
+    return _old;
+  }
+}
+
+inline ZForwarding* ZGeneration::forwarding(zaddress_unsafe addr) const {
+  return _forwarding_table.get(addr);
+}
+
+inline bool ZGeneration::should_worker_resize() {
+  return _workers.should_worker_resize();
+}
+
+inline ZStatHeap* ZGeneration::stat_heap() {
+  return &_stat_heap;
+}
+
+inline ZStatCycle* ZGeneration::stat_cycle() {
+  return &_stat_cycle;
+}
+
+inline ZStatWorkers* ZGeneration::stat_workers() {
+  return &_stat_workers;
+}
+
+inline ZStatMark* ZGeneration::stat_mark() {
+  return &_stat_mark;
+}
+
+inline ZStatRelocation* ZGeneration::stat_relocation() {
+  return &_stat_relocation;
+}
+
+inline ZPageTable* ZGeneration::page_table() const {
+  return _page_table;
+}
+
+inline const ZForwardingTable* ZGeneration::forwarding_table() const {
+  return &_forwarding_table;
+}
+
+template <bool resurrect, bool gc_thread, bool follow, bool finalizable>
+inline void ZGeneration::mark_object(zaddress addr) {
+  assert(is_phase_mark(), "Should be marking");
+  _mark.mark_object<resurrect, gc_thread, follow, finalizable>(addr);
+}
+
+template <bool resurrect, bool gc_thread, bool follow, bool finalizable>
+inline void ZGeneration::mark_object_if_active(zaddress addr) {
+  if (is_phase_mark()) {
+    mark_object<resurrect, gc_thread, follow, finalizable>(addr);
+  }
+}
+
+inline zaddress ZGeneration::relocate_or_remap_object(zaddress_unsafe addr) {
+  ZForwarding* const forwarding = _forwarding_table.get(addr);
+  if (forwarding == nullptr) {
+    // Not forwarding
+    return safe(addr);
+  }
+
+  // Relocate object
+  return _relocate.relocate_object(forwarding, addr);
+}
+
+inline zaddress ZGeneration::remap_object(zaddress_unsafe addr) {
+  ZForwarding* const forwarding = _forwarding_table.get(addr);
+  if (forwarding == nullptr) {
+    // Not forwarding
+    return safe(addr);
+  }
+
+  // Remap object
+  return _relocate.forward_object(forwarding, addr);
+}
+
+inline ZYoungType ZGenerationYoung::type() const {
+  assert(_active_type != ZYoungType::none, "Invalid type");
+  return _active_type;
+}
+
+inline void ZGenerationYoung::remember(volatile zpointer* p) {
+  _remembered.remember(p);
+}
+
+inline void ZGenerationYoung::scan_remembered_field(volatile zpointer* p) {
+  _remembered.scan_field(p);
+}
+
+inline bool ZGenerationYoung::is_remembered(volatile zpointer* p) const {
+  return _remembered.is_remembered(p);
+}
+
+inline ReferenceDiscoverer* ZGenerationOld::reference_discoverer() {
+  return &_reference_processor;
+}
+
+inline bool ZGenerationOld::active_remset_is_current() const {
+  assert(_young_seqnum_at_reloc_start != 0, "Must be set before used");
+
+  // The remembered set bits flip every time a new young collection starts
+  const uint32_t seqnum = ZGeneration::young()->seqnum();
+  const uint32_t seqnum_diff = seqnum - _young_seqnum_at_reloc_start;
+  const bool in_current = (seqnum_diff & 1u) == 0u;
+  return in_current;
+}
+
+inline ZRelocateQueue* ZGenerationOld::relocate_queue() {
+  return _relocate.queue();
+}
+
+#endif // SHARE_GC_Z_ZGENERATION_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zGenerationId.hpp b/src/hotspot/share/gc/z/zGenerationId.hpp
new file mode 100644
index 00000000000..a93d269bbb5
--- /dev/null
+++ b/src/hotspot/share/gc/z/zGenerationId.hpp
@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZGENERATIONID_HPP
+#define SHARE_GC_Z_ZGENERATIONID_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+enum class ZGenerationId : uint8_t {
+  young,
+  old
+};
+
+enum class ZGenerationIdOptional : uint8_t {
+  young,
+  old,
+  none
+};
+
+#endif // SHARE_GC_Z_ZGENERATIONID_HPP
diff --git a/src/hotspot/share/gc/z/zGlobals.cpp b/src/hotspot/share/gc/z/zGlobals.cpp
index 28200e23b6e..460ed4f09fe 100644
--- a/src/hotspot/share/gc/z/zGlobals.cpp
+++ b/src/hotspot/share/gc/z/zGlobals.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,9 +24,6 @@
 #include "precompiled.hpp"
 #include "gc/z/zGlobals.hpp"
 
-uint32_t   ZGlobalPhase                = ZPhaseRelocate;
-uint32_t   ZGlobalSeqNum               = 1;
-
 size_t     ZPageSizeMediumShift;
 size_t     ZPageSizeMedium;
 
@@ -37,43 +34,3 @@ int        ZObjectAlignmentMediumShift;
 
 const int& ZObjectAlignmentSmall       = MinObjAlignmentInBytes;
 int        ZObjectAlignmentMedium;
-
-uintptr_t  ZAddressGoodMask;
-uintptr_t  ZAddressBadMask;
-uintptr_t  ZAddressWeakBadMask;
-
-static uint32_t* ZAddressCalculateBadMaskHighOrderBitsAddr() {
-  const uintptr_t addr = reinterpret_cast<uintptr_t>(&ZAddressBadMask);
-  return reinterpret_cast<uint32_t*>(addr + ZAddressBadMaskHighOrderBitsOffset);
-}
-
-uint32_t*  ZAddressBadMaskHighOrderBitsAddr = ZAddressCalculateBadMaskHighOrderBitsAddr();
-
-size_t     ZAddressOffsetBits;
-uintptr_t  ZAddressOffsetMask;
-size_t     ZAddressOffsetMax;
-
-size_t     ZAddressMetadataShift;
-uintptr_t  ZAddressMetadataMask;
-
-uintptr_t  ZAddressMetadataMarked;
-uintptr_t  ZAddressMetadataMarked0;
-uintptr_t  ZAddressMetadataMarked1;
-uintptr_t  ZAddressMetadataRemapped;
-uintptr_t  ZAddressMetadataFinalizable;
-
-const char* ZGlobalPhaseToString() {
-  switch (ZGlobalPhase) {
-  case ZPhaseMark:
-    return "Mark";
-
-  case ZPhaseMarkCompleted:
-    return "MarkCompleted";
-
-  case ZPhaseRelocate:
-    return "Relocate";
-
-  default:
-    return "Unknown";
-  }
-}
diff --git a/src/hotspot/share/gc/z/zGlobals.hpp b/src/hotspot/share/gc/z/zGlobals.hpp
index 3b65c75076a..0ca08942080 100644
--- a/src/hotspot/share/gc/z/zGlobals.hpp
+++ b/src/hotspot/share/gc/z/zGlobals.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,31 +31,13 @@
 // Collector name
 const char* const ZName                         = "The Z Garbage Collector";
 
-// Global phase state
-extern uint32_t   ZGlobalPhase;
-const uint32_t    ZPhaseMark                    = 0;
-const uint32_t    ZPhaseMarkCompleted           = 1;
-const uint32_t    ZPhaseRelocate                = 2;
-const char*       ZGlobalPhaseToString();
-
-// Global sequence number
-extern uint32_t   ZGlobalSeqNum;
-
 // Granule shift/size
 const size_t      ZGranuleSizeShift             = 21; // 2MB
 const size_t      ZGranuleSize                  = (size_t)1 << ZGranuleSizeShift;
 
-// Number of heap views
-const size_t      ZHeapViews                    = ZPlatformHeapViews;
-
 // Virtual memory to physical memory ratio
 const size_t      ZVirtualToPhysicalRatio       = 16; // 16:1
 
-// Page types
-const uint8_t     ZPageTypeSmall                = 0;
-const uint8_t     ZPageTypeMedium               = 1;
-const uint8_t     ZPageTypeLarge                = 2;
-
 // Page size shifts
 const size_t      ZPageSizeSmallShift           = ZGranuleSizeShift;
 extern size_t     ZPageSizeMediumShift;
@@ -78,53 +60,11 @@ extern const int& ZObjectAlignmentSmall;
 extern int        ZObjectAlignmentMedium;
 const int         ZObjectAlignmentLarge         = 1 << ZObjectAlignmentLargeShift;
 
-//
-// Good/Bad mask states
-// --------------------
-//
-//                 GoodMask         BadMask          WeakGoodMask     WeakBadMask
-//                 --------------------------------------------------------------
-//  Marked0        001              110              101              010
-//  Marked1        010              101              110              001
-//  Remapped       100              011              100              011
-//
-
-// Good/bad masks
-extern uintptr_t  ZAddressGoodMask;
-extern uintptr_t  ZAddressBadMask;
-extern uintptr_t  ZAddressWeakBadMask;
-
-// The bad mask is 64 bit. Its high order 32 bits contain all possible value combinations
-// that this mask will have. Therefore, the memory where the 32 high order bits are stored,
-// can be used as a 32 bit GC epoch counter, that has a different bit pattern every time
-// the bad mask is flipped. This provides a pointer to said 32 bits.
-extern uint32_t*  ZAddressBadMaskHighOrderBitsAddr;
-const int         ZAddressBadMaskHighOrderBitsOffset = LITTLE_ENDIAN_ONLY(4) BIG_ENDIAN_ONLY(0);
-
-// Pointer part of address
-extern size_t     ZAddressOffsetBits;
-const  size_t     ZAddressOffsetShift           = 0;
-extern uintptr_t  ZAddressOffsetMask;
-extern size_t     ZAddressOffsetMax;
-
-// Metadata part of address
-const size_t      ZAddressMetadataBits          = 4;
-extern size_t     ZAddressMetadataShift;
-extern uintptr_t  ZAddressMetadataMask;
-
-// Metadata types
-extern uintptr_t  ZAddressMetadataMarked;
-extern uintptr_t  ZAddressMetadataMarked0;
-extern uintptr_t  ZAddressMetadataMarked1;
-extern uintptr_t  ZAddressMetadataRemapped;
-extern uintptr_t  ZAddressMetadataFinalizable;
-
 // Cache line size
 const size_t      ZCacheLineSize                = ZPlatformCacheLineSize;
 #define           ZCACHE_ALIGNED                ATTRIBUTE_ALIGNED(ZCacheLineSize)
 
 // Mark stack space
-extern uintptr_t  ZMarkStackSpaceStart;
 const size_t      ZMarkStackSpaceExpandSize     = (size_t)1 << 25; // 32M
 
 // Mark stack and magazine sizes
@@ -147,10 +87,10 @@ const size_t      ZMarkCacheSize                = 1024; // Must be a power of tw
 // Partial array minimum size
 const size_t      ZMarkPartialArrayMinSizeShift = 12; // 4K
 const size_t      ZMarkPartialArrayMinSize      = (size_t)1 << ZMarkPartialArrayMinSizeShift;
+const size_t      ZMarkPartialArrayMinLength    = ZMarkPartialArrayMinSize / oopSize;
 
 // Max number of proactive/terminate flush attempts
 const size_t      ZMarkProactiveFlushMax        = 10;
-const size_t      ZMarkTerminateFlushMax        = 3;
 
 // Try complete mark timeout
 const uint64_t    ZMarkCompleteTimeout          = 200; // us
diff --git a/src/hotspot/share/gc/z/zGranuleMap.hpp b/src/hotspot/share/gc/z/zGranuleMap.hpp
index f2cb317c8de..58c95e331b6 100644
--- a/src/hotspot/share/gc/z/zGranuleMap.hpp
+++ b/src/hotspot/share/gc/z/zGranuleMap.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,34 +24,41 @@
 #ifndef SHARE_GC_Z_ZGRANULEMAP_HPP
 #define SHARE_GC_Z_ZGRANULEMAP_HPP
 
+#include "gc/z/zAddress.hpp"
 #include "gc/z/zArray.hpp"
 #include "memory/allocation.hpp"
 
 template <typename T>
 class ZGranuleMap {
   friend class VMStructs;
-  template <typename> friend class ZGranuleMapIterator;
+  template <typename, bool> friend class ZGranuleMapIterator;
+  friend class ZForwardingTable;
+  friend class ZPageTable;
+  friend class ZRemsetTableIterator;
 
 private:
   const size_t _size;
   T* const     _map;
 
-  size_t index_for_offset(uintptr_t offset) const;
+  size_t index_for_offset(zoffset offset) const;
+
+  T at(size_t index) const;
 
 public:
   ZGranuleMap(size_t max_offset);
   ~ZGranuleMap();
 
-  T get(uintptr_t offset) const;
-  void put(uintptr_t offset, T value);
-  void put(uintptr_t offset, size_t size, T value);
+  T get(zoffset offset) const;
+  void put(zoffset offset, T value);
+  void put(zoffset offset, size_t size, T value);
 
-  T get_acquire(uintptr_t offset) const;
-  void release_put(uintptr_t offset, T value);
+  T get_acquire(zoffset offset) const;
+  void release_put(zoffset offset, T value);
+  void release_put(zoffset offset, size_t size, T value);
 };
 
-template <typename T>
-class ZGranuleMapIterator : public ZArrayIteratorImpl<T, false /* Parallel */> {
+template <typename T, bool Parallel>
+class ZGranuleMapIterator : public ZArrayIteratorImpl<T, Parallel> {
 public:
   ZGranuleMapIterator(const ZGranuleMap<T>* granule_map);
 };
diff --git a/src/hotspot/share/gc/z/zGranuleMap.inline.hpp b/src/hotspot/share/gc/z/zGranuleMap.inline.hpp
index 457bccfa61c..8265ac98928 100644
--- a/src/hotspot/share/gc/z/zGranuleMap.inline.hpp
+++ b/src/hotspot/share/gc/z/zGranuleMap.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
 
 #include "gc/z/zGranuleMap.hpp"
 
+#include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zArray.inline.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "memory/allocation.inline.hpp"
@@ -46,49 +47,62 @@ inline ZGranuleMap<T>::~ZGranuleMap() {
 }
 
 template <typename T>
-inline size_t ZGranuleMap<T>::index_for_offset(uintptr_t offset) const {
-  const size_t index = offset >> ZGranuleSizeShift;
+inline size_t ZGranuleMap<T>::index_for_offset(zoffset offset) const {
+  const size_t index = untype(offset) >> ZGranuleSizeShift;
   assert(index < _size, "Invalid index");
+
   return index;
 }
 
 template <typename T>
-inline T ZGranuleMap<T>::get(uintptr_t offset) const {
-  const size_t index = index_for_offset(offset);
-  return _map[index];
+inline T ZGranuleMap<T>::at(size_t index) const {
+  assert(index < _size, "Invalid index");
+  return Atomic::load(_map + index);
 }
 
 template <typename T>
-inline void ZGranuleMap<T>::put(uintptr_t offset, T value) {
+inline T ZGranuleMap<T>::get(zoffset offset) const {
   const size_t index = index_for_offset(offset);
-  _map[index] = value;
+  return at(index);
 }
 
 template <typename T>
-inline void ZGranuleMap<T>::put(uintptr_t offset, size_t size, T value) {
+inline void ZGranuleMap<T>::put(zoffset offset, T value) {
+  const size_t index = index_for_offset(offset);
+  Atomic::store(_map + index, value);
+}
+
+template <typename T>
+inline void ZGranuleMap<T>::put(zoffset offset, size_t size, T value) {
   assert(is_aligned(size, ZGranuleSize), "Misaligned");
 
   const size_t start_index = index_for_offset(offset);
   const size_t end_index = start_index + (size >> ZGranuleSizeShift);
   for (size_t index = start_index; index < end_index; index++) {
-    _map[index] = value;
+    Atomic::store(_map + index, value);
   }
 }
 
 template <typename T>
-inline T ZGranuleMap<T>::get_acquire(uintptr_t offset) const {
+inline T ZGranuleMap<T>::get_acquire(zoffset offset) const {
   const size_t index = index_for_offset(offset);
   return Atomic::load_acquire(_map + index);
 }
 
 template <typename T>
-inline void ZGranuleMap<T>::release_put(uintptr_t offset, T value) {
+inline void ZGranuleMap<T>::release_put(zoffset offset, T value) {
   const size_t index = index_for_offset(offset);
   Atomic::release_store(_map + index, value);
 }
 
 template <typename T>
-inline ZGranuleMapIterator<T>::ZGranuleMapIterator(const ZGranuleMap<T>* granule_map) :
-    ZArrayIteratorImpl<T, false /* Parallel */>(granule_map->_map, granule_map->_size) {}
+inline void ZGranuleMap<T>::release_put(zoffset offset, size_t size, T value) {
+  OrderAccess::release();
+  put(offset, size, value);
+}
+
+template <typename T, bool Parallel>
+inline ZGranuleMapIterator<T, Parallel>::ZGranuleMapIterator(const ZGranuleMap<T>* granule_map) :
+    ZArrayIteratorImpl<T, Parallel>(granule_map->_map, granule_map->_size) {}
 
 #endif // SHARE_GC_Z_ZGRANULEMAP_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zHash.hpp b/src/hotspot/share/gc/z/zHash.hpp
index e6469698488..d8903c6f99a 100644
--- a/src/hotspot/share/gc/z/zHash.hpp
+++ b/src/hotspot/share/gc/z/zHash.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,13 +24,15 @@
 #ifndef SHARE_GC_Z_ZHASH_HPP
 #define SHARE_GC_Z_ZHASH_HPP
 
-#include "memory/allStatic.hpp"
+#include "gc/z/zAddress.hpp"
+#include "memory/allocation.hpp"
 #include "utilities/globalDefinitions.hpp"
 
 class ZHash : public AllStatic {
 public:
   static uint32_t uint32_to_uint32(uint32_t key);
   static uint32_t address_to_uint32(uintptr_t key);
+  static uint32_t offset_to_uint32(zoffset key);
 };
 
 #endif // SHARE_GC_Z_ZHASH_HPP
diff --git a/src/hotspot/share/gc/z/zHash.inline.hpp b/src/hotspot/share/gc/z/zHash.inline.hpp
index 2fda5189da4..987501219fb 100644
--- a/src/hotspot/share/gc/z/zHash.inline.hpp
+++ b/src/hotspot/share/gc/z/zHash.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -74,4 +74,8 @@ inline uint32_t ZHash::address_to_uint32(uintptr_t key) {
   return uint32_to_uint32((uint32_t)(key >> 3));
 }
 
+inline uint32_t ZHash::offset_to_uint32(zoffset key) {
+  return address_to_uint32(untype(key));
+}
+
 #endif // SHARE_GC_Z_ZHASH_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zHeap.cpp b/src/hotspot/share/gc/z/zHeap.cpp
index 8ad31a60738..7c2eb9707b2 100644
--- a/src/hotspot/share/gc/z/zHeap.cpp
+++ b/src/hotspot/share/gc/z/zHeap.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,62 +24,78 @@
 #include "precompiled.hpp"
 #include "classfile/classLoaderDataGraph.hpp"
 #include "gc/shared/gc_globals.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
 #include "gc/shared/locationPrinter.hpp"
 #include "gc/shared/tlab_globals.hpp"
 #include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zArray.inline.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zHeapIterator.hpp"
 #include "gc/z/zHeuristics.hpp"
-#include "gc/z/zMark.inline.hpp"
 #include "gc/z/zPage.inline.hpp"
 #include "gc/z/zPageTable.inline.hpp"
-#include "gc/z/zRelocationSet.inline.hpp"
-#include "gc/z/zRelocationSetSelector.inline.hpp"
 #include "gc/z/zResurrection.hpp"
 #include "gc/z/zStat.hpp"
-#include "gc/z/zThread.inline.hpp"
+#include "gc/z/zUncoloredRoot.inline.hpp"
+#include "gc/z/zUtils.hpp"
 #include "gc/z/zVerify.hpp"
 #include "gc/z/zWorkers.hpp"
 #include "logging/log.hpp"
 #include "memory/iterator.hpp"
 #include "memory/metaspaceUtils.hpp"
 #include "memory/resourceArea.hpp"
-#include "prims/jvmtiTagMap.hpp"
-#include "runtime/handshake.hpp"
 #include "runtime/javaThread.hpp"
-#include "runtime/safepoint.hpp"
 #include "utilities/debug.hpp"
 
 static const ZStatCounter ZCounterUndoPageAllocation("Memory", "Undo Page Allocation", ZStatUnitOpsPerSecond);
 static const ZStatCounter ZCounterOutOfMemory("Memory", "Out Of Memory", ZStatUnitOpsPerSecond);
 
-ZHeap* ZHeap::_heap = NULL;
+ZHeap* ZHeap::_heap = nullptr;
 
 ZHeap::ZHeap() :
-    _workers(),
-    _object_allocator(),
-    _page_allocator(&_workers, MinHeapSize, InitialHeapSize, MaxHeapSize),
+    _page_allocator(MinHeapSize, InitialHeapSize, SoftMaxHeapSize, MaxHeapSize),
     _page_table(),
-    _forwarding_table(),
-    _mark(&_workers, &_page_table),
-    _reference_processor(&_workers),
-    _weak_roots_processor(&_workers),
-    _relocate(&_workers),
-    _relocation_set(&_workers),
-    _unload(&_workers),
-    _serviceability(min_capacity(), max_capacity()) {
+    _allocator_eden(),
+    _allocator_relocation(),
+    _serviceability(initial_capacity(), min_capacity(), max_capacity()),
+    _old(&_page_table, &_page_allocator),
+    _young(&_page_table, _old.forwarding_table(), &_page_allocator),
+    _initialized(false) {
+
   // Install global heap instance
-  assert(_heap == NULL, "Already initialized");
+  assert(_heap == nullptr, "Already initialized");
   _heap = this;
 
+  if (!_page_allocator.is_initialized() || !_young.is_initialized() || !_old.is_initialized()) {
+    return;
+  }
+
+  // Prime cache
+  if (!_page_allocator.prime_cache(_old.workers(), InitialHeapSize)) {
+    log_error_p(gc)("Failed to allocate initial Java heap (" SIZE_FORMAT "M)", InitialHeapSize / M);
+    return;
+  }
+
+  if (UseDynamicNumberOfGCThreads) {
+    log_info_p(gc, init)("GC Workers Max: %u (dynamic)", ConcGCThreads);
+  }
+
   // Update statistics
-  ZStatHeap::set_at_initialize(_page_allocator.stats());
+  _young.stat_heap()->at_initialize(_page_allocator.min_capacity(), _page_allocator.max_capacity());
+  _old.stat_heap()->at_initialize(_page_allocator.min_capacity(), _page_allocator.max_capacity());
+
+  // Successfully initialized
+  _initialized = true;
 }
 
 bool ZHeap::is_initialized() const {
-  return _page_allocator.is_initialized() && _mark.is_initialized();
+  return _initialized;
+}
+
+size_t ZHeap::initial_capacity() const {
+  return _page_allocator.initial_capacity();
 }
 
 size_t ZHeap::min_capacity() const {
@@ -102,6 +118,18 @@ size_t ZHeap::used() const {
   return _page_allocator.used();
 }
 
+size_t ZHeap::used_generation(ZGenerationId id) const {
+  return _page_allocator.used_generation(id);
+}
+
+size_t ZHeap::used_young() const {
+  return _page_allocator.used_generation(ZGenerationId::young);
+}
+
+size_t ZHeap::used_old() const {
+  return _page_allocator.used_generation(ZGenerationId::old);
+}
+
 size_t ZHeap::unused() const {
   return _page_allocator.unused();
 }
@@ -111,7 +139,7 @@ size_t ZHeap::tlab_capacity() const {
 }
 
 size_t ZHeap::tlab_used() const {
-  return _object_allocator.used();
+  return _allocator_eden.tlab_used();
 }
 
 size_t ZHeap::max_tlab_size() const {
@@ -119,7 +147,7 @@ size_t ZHeap::max_tlab_size() const {
 }
 
 size_t ZHeap::unsafe_max_tlab_alloc() const {
-  size_t size = _object_allocator.remaining();
+  size_t size = _allocator_eden.remaining();
 
   if (size < MinTLABSize) {
     // The remaining space in the allocator is not enough to
@@ -134,33 +162,58 @@ size_t ZHeap::unsafe_max_tlab_alloc() const {
 }
 
 bool ZHeap::is_in(uintptr_t addr) const {
+  if (addr == 0) {
+    // Null isn't in the heap.
+    return false;
+  }
+
   // An address is considered to be "in the heap" if it points into
   // the allocated part of a page, regardless of which heap view is
   // used. Note that an address with the finalizable metadata bit set
   // is not pointing into a heap view, and therefore not considered
   // to be "in the heap".
 
-  if (ZAddress::is_in(addr)) {
-    const ZPage* const page = _page_table.get(addr);
-    if (page != NULL) {
-      return page->is_in(addr);
+  assert(!is_valid(zpointer(addr)), "Don't pass in colored oops");
+
+  if (!is_valid(zaddress(addr))) {
+    return false;
+  }
+
+  const zaddress o = to_zaddress(addr);
+  const ZPage* const page = _page_table.get(o);
+  if (page == nullptr) {
+    return false;
+  }
+
+  return is_in_page_relaxed(page, o);
+}
+
+bool ZHeap::is_in_page_relaxed(const ZPage* page, zaddress addr) const {
+  if (page->is_in(addr)) {
+    return true;
+  }
+
+  // Could still be a from-object during an in-place relocation
+  if (_old.is_phase_relocate()) {
+    const ZForwarding* const forwarding = _old.forwarding(unsafe(addr));
+    if (forwarding != nullptr && forwarding->in_place_relocation_is_below_top_at_start(ZAddress::offset(addr))) {
+      return true;
+    }
+  }
+  if (_young.is_phase_relocate()) {
+    const ZForwarding* const forwarding = _young.forwarding(unsafe(addr));
+    if (forwarding != nullptr && forwarding->in_place_relocation_is_below_top_at_start(ZAddress::offset(addr))) {
+      return true;
     }
   }
 
   return false;
 }
 
-uint ZHeap::active_workers() const {
-  return _workers.active_workers();
-}
-
-void ZHeap::set_active_workers(uint nworkers) {
-  _workers.set_active_workers(nworkers);
-}
-
 void ZHeap::threads_do(ThreadClosure* tc) const {
   _page_allocator.threads_do(tc);
-  _workers.threads_do(tc);
+  _young.threads_do(tc);
+  _old.threads_do(tc);
 }
 
 void ZHeap::out_of_memory() {
@@ -170,9 +223,9 @@ void ZHeap::out_of_memory() {
   log_info(gc)("Out Of Memory (%s)", Thread::current()->name());
 }
 
-ZPage* ZHeap::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) {
-  ZPage* const page = _page_allocator.alloc_page(type, size, flags);
-  if (page != NULL) {
+ZPage* ZHeap::alloc_page(ZPageType type, size_t size, ZAllocationFlags flags, ZPageAge age) {
+  ZPage* const page = _page_allocator.alloc_page(type, size, flags, age);
+  if (page != nullptr) {
     // Insert page table entry
     _page_table.insert(page);
   }
@@ -185,276 +238,69 @@ void ZHeap::undo_alloc_page(ZPage* page) {
 
   ZStatInc(ZCounterUndoPageAllocation);
   log_trace(gc)("Undo page allocation, thread: " PTR_FORMAT " (%s), page: " PTR_FORMAT ", size: " SIZE_FORMAT,
-                ZThread::id(), ZThread::name(), p2i(page), page->size());
+                p2i(Thread::current()), ZUtils::thread_name(), p2i(page), page->size());
 
-  free_page(page, false /* reclaimed */);
+  free_page(page);
 }
 
-void ZHeap::free_page(ZPage* page, bool reclaimed) {
+void ZHeap::free_page(ZPage* page) {
   // Remove page table entry
   _page_table.remove(page);
 
+  if (page->is_old()) {
+    page->verify_remset_cleared_current();
+    page->verify_remset_cleared_previous();
+  }
+
   // Free page
-  _page_allocator.free_page(page, reclaimed);
+  _page_allocator.free_page(page);
 }
 
-void ZHeap::free_pages(const ZArray<ZPage*>* pages, bool reclaimed) {
+size_t ZHeap::free_empty_pages(const ZArray<ZPage*>* pages) {
+  size_t freed = 0;
   // Remove page table entries
   ZArrayIterator<ZPage*> iter(pages);
   for (ZPage* page; iter.next(&page);) {
+    if (page->is_old()) {
+      // The remset of pages should be clean when installed into the page
+      // cache.
+      page->remset_clear();
+    }
     _page_table.remove(page);
+    freed += page->size();
   }
 
   // Free pages
-  _page_allocator.free_pages(pages, reclaimed);
-}
+  _page_allocator.free_pages(pages);
 
-void ZHeap::flip_to_marked() {
-  ZVerifyViewsFlip flip(&_page_allocator);
-  ZAddress::flip_to_marked();
-}
-
-void ZHeap::flip_to_remapped() {
-  ZVerifyViewsFlip flip(&_page_allocator);
-  ZAddress::flip_to_remapped();
-}
-
-void ZHeap::mark_start() {
-  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
-
-  // Verification
-  ClassLoaderDataGraph::verify_claimed_marks_cleared(ClassLoaderData::_claim_strong);
-
-  if (ZHeap::heap()->has_alloc_stalled()) {
-    // If there are stalled allocations, ensure that regardless of the
-    // cause of the GC, we have to clear soft references, as we are just
-    // about to increment the sequence number, and all previous allocations
-    // will throw if not presented with enough memory.
-    ZHeap::heap()->set_soft_reference_policy(true);
-  }
-
-  // Flip address view
-  flip_to_marked();
-
-  // Retire allocating pages
-  _object_allocator.retire_pages();
-
-  // Reset allocated/reclaimed/used statistics
-  _page_allocator.reset_statistics();
-
-  // Reset encountered/dropped/enqueued statistics
-  _reference_processor.reset_statistics();
-
-  // Enter mark phase
-  ZGlobalPhase = ZPhaseMark;
-
-  // Reset marking information and mark roots
-  _mark.start();
-
-  // Update statistics
-  ZStatHeap::set_at_mark_start(_page_allocator.stats());
-}
-
-void ZHeap::mark(bool initial) {
-  _mark.mark(initial);
-}
-
-void ZHeap::mark_flush_and_free(Thread* thread) {
-  _mark.flush_and_free(thread);
-}
-
-bool ZHeap::mark_end() {
-  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
-
-  // Try end marking
-  if (!_mark.end()) {
-    // Marking not completed, continue concurrent mark
-    return false;
-  }
-
-  // Enter mark completed phase
-  ZGlobalPhase = ZPhaseMarkCompleted;
-
-  // Verify after mark
-  ZVerify::after_mark();
-
-  // Update statistics
-  ZStatHeap::set_at_mark_end(_page_allocator.stats());
-
-  // Block resurrection of weak/phantom references
-  ZResurrection::block();
-
-  // Prepare to unload stale metadata and nmethods
-  _unload.prepare();
-
-  // Notify JVMTI that some tagmap entry objects may have died.
-  JvmtiTagMap::set_needs_cleaning();
-
-  return true;
-}
-
-void ZHeap::mark_free() {
-  _mark.free();
+  return freed;
 }
 
 void ZHeap::keep_alive(oop obj) {
-  ZBarrier::keep_alive_barrier_on_oop(obj);
+  const zaddress addr = to_zaddress(obj);
+  ZBarrier::mark<ZMark::Resurrect, ZMark::AnyThread, ZMark::Follow, ZMark::Strong>(addr);
 }
 
-void ZHeap::set_soft_reference_policy(bool clear) {
-  _reference_processor.set_soft_reference_policy(clear);
+void ZHeap::mark_flush_and_free(Thread* thread) {
+  _young.mark_flush_and_free(thread);
+  _old.mark_flush_and_free(thread);
 }
 
-class ZRendezvousClosure : public HandshakeClosure {
-public:
-  ZRendezvousClosure() :
-      HandshakeClosure("ZRendezvous") {}
-
-  void do_thread(Thread* thread) {}
-};
-
-void ZHeap::process_non_strong_references() {
-  // Process Soft/Weak/Final/PhantomReferences
-  _reference_processor.process_references();
-
-  // Process weak roots
-  _weak_roots_processor.process_weak_roots();
-
-  // Unlink stale metadata and nmethods
-  _unload.unlink();
-
-  // Perform a handshake. This is needed 1) to make sure that stale
-  // metadata and nmethods are no longer observable. And 2), to
-  // prevent the race where a mutator first loads an oop, which is
-  // logically null but not yet cleared. Then this oop gets cleared
-  // by the reference processor and resurrection is unblocked. At
-  // this point the mutator could see the unblocked state and pass
-  // this invalid oop through the normal barrier path, which would
-  // incorrectly try to mark the oop.
-  ZRendezvousClosure cl;
-  Handshake::execute(&cl);
-
-  // Unblock resurrection of weak/phantom references
-  ZResurrection::unblock();
-
-  // Purge stale metadata and nmethods that were unlinked
-  _unload.purge();
-
-  // Enqueue Soft/Weak/Final/PhantomReferences. Note that this
-  // must be done after unblocking resurrection. Otherwise the
-  // Finalizer thread could call Reference.get() on the Finalizers
-  // that were just enqueued, which would incorrectly return null
-  // during the resurrection block window, since such referents
-  // are only Finalizable marked.
-  _reference_processor.enqueue_references();
-
-  // Clear old markings claim bits.
-  // Note: Clearing _claim_strong also clears _claim_finalizable.
-  ClassLoaderDataGraph::clear_claimed_marks(ClassLoaderData::_claim_strong);
-}
-
-void ZHeap::free_empty_pages(ZRelocationSetSelector* selector, int bulk) {
-  // Freeing empty pages in bulk is an optimization to avoid grabbing
-  // the page allocator lock, and trying to satisfy stalled allocations
-  // too frequently.
-  if (selector->should_free_empty_pages(bulk)) {
-    free_pages(selector->empty_pages(), true /* reclaimed */);
-    selector->clear_empty_pages();
-  }
-}
-
-void ZHeap::select_relocation_set() {
-  // Do not allow pages to be deleted
-  _page_allocator.enable_deferred_delete();
-
-  // Register relocatable pages with selector
-  ZRelocationSetSelector selector;
-  ZPageTableIterator pt_iter(&_page_table);
-  for (ZPage* page; pt_iter.next(&page);) {
-    if (!page->is_relocatable()) {
-      // Not relocatable, don't register
-      continue;
-    }
-
-    if (page->is_marked()) {
-      // Register live page
-      selector.register_live_page(page);
-    } else {
-      // Register empty page
-      selector.register_empty_page(page);
-
-      // Reclaim empty pages in bulk
-      free_empty_pages(&selector, 64 /* bulk */);
-    }
-  }
-
-  // Reclaim remaining empty pages
-  free_empty_pages(&selector, 0 /* bulk */);
-
-  // Allow pages to be deleted
-  _page_allocator.disable_deferred_delete();
-
-  // Select relocation set
-  selector.select();
-
-  // Install relocation set
-  _relocation_set.install(&selector);
-
-  // Setup forwarding table
-  ZRelocationSetIterator rs_iter(&_relocation_set);
-  for (ZForwarding* forwarding; rs_iter.next(&forwarding);) {
-    _forwarding_table.insert(forwarding);
-  }
-
-  // Update statistics
-  ZStatRelocation::set_at_select_relocation_set(selector.stats());
-  ZStatHeap::set_at_select_relocation_set(selector.stats());
-}
-
-void ZHeap::reset_relocation_set() {
-  // Reset forwarding table
-  ZRelocationSetIterator iter(&_relocation_set);
-  for (ZForwarding* forwarding; iter.next(&forwarding);) {
-    _forwarding_table.remove(forwarding);
-  }
-
-  // Reset relocation set
-  _relocation_set.reset();
-}
-
-void ZHeap::relocate_start() {
-  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
-
-  // Finish unloading stale metadata and nmethods
-  _unload.finish();
-
-  // Flip address view
-  flip_to_remapped();
-
-  // Enter relocate phase
-  ZGlobalPhase = ZPhaseRelocate;
-
-  // Update statistics
-  ZStatHeap::set_at_relocate_start(_page_allocator.stats());
-}
-
-void ZHeap::relocate() {
-  // Relocate relocation set
-  _relocate.relocate(&_relocation_set);
-
-  // Update statistics
-  ZStatHeap::set_at_relocate_end(_page_allocator.stats(), _object_allocator.relocated());
-}
-
-bool ZHeap::is_allocating(uintptr_t addr) const {
+bool ZHeap::is_allocating(zaddress addr) const {
   const ZPage* const page = _page_table.get(addr);
   return page->is_allocating();
 }
 
-void ZHeap::object_iterate(ObjectClosure* cl, bool visit_weaks) {
+void ZHeap::object_iterate(ObjectClosure* object_cl, bool visit_weaks) {
   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
   ZHeapIterator iter(1 /* nworkers */, visit_weaks);
-  iter.object_iterate(cl, 0 /* worker_id */);
+  iter.object_iterate(object_cl, 0 /* worker_id */);
+}
+
+void ZHeap::object_and_field_iterate(ObjectClosure* object_cl, OopFieldClosure* field_cl, bool visit_weaks) {
+  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
+  ZHeapIterator iter(1 /* nworkers */, visit_weaks);
+  iter.object_and_field_iterate(object_cl, field_cl, 0 /* worker_id */);
 }
 
 ParallelObjectIteratorImpl* ZHeap::parallel_object_iterator(uint nworkers, bool visit_weaks) {
@@ -462,28 +308,20 @@ ParallelObjectIteratorImpl* ZHeap::parallel_object_iterator(uint nworkers, bool
   return new ZHeapIterator(nworkers, visit_weaks);
 }
 
-void ZHeap::pages_do(ZPageClosure* cl) {
-  ZPageTableIterator iter(&_page_table);
-  for (ZPage* page; iter.next(&page);) {
-    cl->do_page(page);
-  }
-  _page_allocator.pages_do(cl);
-}
-
 void ZHeap::serviceability_initialize() {
   _serviceability.initialize();
 }
 
-GCMemoryManager* ZHeap::serviceability_cycle_memory_manager() {
-  return _serviceability.cycle_memory_manager();
+GCMemoryManager* ZHeap::serviceability_cycle_memory_manager(bool minor) {
+  return _serviceability.cycle_memory_manager(minor);
 }
 
-GCMemoryManager* ZHeap::serviceability_pause_memory_manager() {
-  return _serviceability.pause_memory_manager();
+GCMemoryManager* ZHeap::serviceability_pause_memory_manager(bool minor) {
+  return _serviceability.pause_memory_manager(minor);
 }
 
-MemoryPool* ZHeap::serviceability_memory_pool() {
-  return _serviceability.memory_pool();
+MemoryPool* ZHeap::serviceability_memory_pool(ZGenerationId id) {
+  return _serviceability.memory_pool(id);
 }
 
 ZServiceabilityCounters* ZHeap::serviceability_counters() {
@@ -503,7 +341,7 @@ void ZHeap::print_extended_on(outputStream* st) const {
   st->cr();
 
   // Do not allow pages to be deleted
-  _page_allocator.enable_deferred_delete();
+  _page_allocator.enable_safe_destroy();
 
   // Print all pages
   st->print_cr("ZGC Page Table:");
@@ -513,24 +351,99 @@ void ZHeap::print_extended_on(outputStream* st) const {
   }
 
   // Allow pages to be deleted
-  _page_allocator.disable_deferred_delete();
+  _page_allocator.disable_safe_destroy();
 }
 
 bool ZHeap::print_location(outputStream* st, uintptr_t addr) const {
-  if (LocationPrinter::is_valid_obj((void*)addr)) {
-    st->print(PTR_FORMAT " is a %s oop: ", addr, ZAddress::is_good(addr) ? "good" : "bad");
-    ZOop::from_address(addr)->print_on(st);
-    return true;
+  // Intentionally unchecked cast
+  const bool uncolored = is_valid(zaddress(addr));
+  const bool colored = is_valid(zpointer(addr));
+  if (colored && uncolored) {
+    // Should not reach here
+    return false;
+  }
+
+  if (colored) {
+    return print_location(st, zpointer(addr));
+  }
+
+  if (uncolored) {
+    return print_location(st, zaddress(addr));
   }
 
   return false;
 }
 
-void ZHeap::verify() {
-  // Heap verification can only be done between mark end and
-  // relocate start. This is the only window where all oop are
-  // good and the whole heap is in a consistent state.
-  guarantee(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase");
+bool ZHeap::print_location(outputStream* st, zaddress addr) const {
+  assert(is_valid(addr), "must be");
 
-  ZVerify::after_weak_processing();
+  st->print(PTR_FORMAT " is a zaddress: ", untype(addr));
+
+  if (addr == zaddress::null) {
+    st->print_raw_cr("NULL");
+    return true;
+  }
+
+  if (!ZHeap::is_in(untype(addr))) {
+    st->print_raw_cr("not in heap");
+    return false;
+  }
+
+  if (LocationPrinter::is_valid_obj((void*)untype(addr))) {
+    to_oop(addr)->print_on(st);
+    return true;
+  }
+
+  ZPage* const page = ZHeap::page(addr);
+  zaddress_unsafe base;
+
+  if (page->is_relocatable() && page->is_marked() && !ZGeneration::generation(page->generation_id())->is_phase_mark()) {
+    base = page->find_base((volatile zpointer*) addr);
+  } else {
+    // TODO: This part is probably broken, but register printing recovers from crashes
+    st->print_raw("Unreliable ");
+    base = page->find_base_unsafe((volatile zpointer*) addr);
+  }
+
+  if (base == zaddress_unsafe::null) {
+    st->print_raw_cr("Cannot find base");
+    return false;
+  }
+
+  if (untype(base) == untype(addr)) {
+    st->print_raw_cr("Bad mark info/base");
+    return false;
+  }
+
+  st->print_raw_cr("Internal address");
+  print_location(st, untype(base));
+  return true;
+}
+
+bool ZHeap::print_location(outputStream* st, zpointer ptr) const {
+  assert(is_valid(ptr), "must be");
+
+  st->print(PTR_FORMAT " is %s zpointer: ", untype(ptr),
+            ZPointer::is_load_good(ptr) ? "a good" : "a bad");
+
+  if (!ZPointer::is_load_good(ptr)) {
+    st->print_cr("decoded " PTR_FORMAT, untype(ZPointer::uncolor_unsafe(ptr)));
+    // ptr is not load good but let us still investigate the uncolored address
+    return print_location(st, untype(ZPointer::uncolor_unsafe(ptr)));
+  }
+
+  const zaddress addr =  ZPointer::uncolor(ptr);
+
+  if (addr == zaddress::null) {
+    st->print_raw_cr("NULL");
+    return true;
+  }
+
+  if (LocationPrinter::is_valid_obj((void*)untype(addr))) {
+    to_oop(addr)->print_on(st);
+    return true;
+  }
+
+  st->print_cr("invalid object " PTR_FORMAT,  untype(addr));
+  return false;
 }
diff --git a/src/hotspot/share/gc/z/zHeap.hpp b/src/hotspot/share/gc/z/zHeap.hpp
index 04250ad551f..38969f1b4f2 100644
--- a/src/hotspot/share/gc/z/zHeap.hpp
+++ b/src/hotspot/share/gc/z/zHeap.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,49 +25,37 @@
 #define SHARE_GC_Z_ZHEAP_HPP
 
 #include "gc/z/zAllocationFlags.hpp"
+#include "gc/z/zAllocator.hpp"
 #include "gc/z/zArray.hpp"
-#include "gc/z/zForwardingTable.hpp"
-#include "gc/z/zMark.hpp"
-#include "gc/z/zObjectAllocator.hpp"
+#include "gc/z/zGeneration.hpp"
+#include "gc/z/zPageAge.hpp"
 #include "gc/z/zPageAllocator.hpp"
 #include "gc/z/zPageTable.hpp"
-#include "gc/z/zReferenceProcessor.hpp"
-#include "gc/z/zRelocate.hpp"
-#include "gc/z/zRelocationSet.hpp"
-#include "gc/z/zWeakRootsProcessor.hpp"
+#include "gc/z/zPageType.hpp"
 #include "gc/z/zServiceability.hpp"
-#include "gc/z/zUnload.hpp"
-#include "gc/z/zWorkers.hpp"
 
-class ThreadClosure;
-class ZPage;
-class ZRelocationSetSelector;
+class OopFieldClosure;
 
 class ZHeap {
+  friend class ZForwardingTest;
+  friend class ZLiveMapTest;
   friend class VMStructs;
 
 private:
-  static ZHeap*       _heap;
+  static ZHeap*           _heap;
 
-  ZWorkers            _workers;
-  ZObjectAllocator    _object_allocator;
-  ZPageAllocator      _page_allocator;
-  ZPageTable          _page_table;
-  ZForwardingTable    _forwarding_table;
-  ZMark               _mark;
-  ZReferenceProcessor _reference_processor;
-  ZWeakRootsProcessor _weak_roots_processor;
-  ZRelocate           _relocate;
-  ZRelocationSet      _relocation_set;
-  ZUnload             _unload;
-  ZServiceability     _serviceability;
+  ZPageAllocator          _page_allocator;
+  ZPageTable              _page_table;
 
-  void flip_to_marked();
-  void flip_to_remapped();
+  ZAllocatorEden          _allocator_eden;
+  ZAllocatorForRelocation _allocator_relocation[ZAllocator::_relocation_allocators];
 
-  void free_empty_pages(ZRelocationSetSelector* selector, int bulk);
+  ZServiceability         _serviceability;
 
-  void out_of_memory();
+  ZGenerationOld          _old;
+  ZGenerationYoung        _young;
+
+  bool                    _initialized;
 
 public:
   static ZHeap* heap();
@@ -76,12 +64,18 @@ public:
 
   bool is_initialized() const;
 
+  void out_of_memory();
+
   // Heap metrics
+  size_t initial_capacity() const;
   size_t min_capacity() const;
   size_t max_capacity() const;
   size_t soft_max_capacity() const;
   size_t capacity() const;
   size_t used() const;
+  size_t used_generation(ZGenerationId id) const;
+  size_t used_young() const;
+  size_t used_old() const;
   size_t unused() const;
 
   size_t tlab_capacity() const;
@@ -90,77 +84,61 @@ public:
   size_t unsafe_max_tlab_alloc() const;
 
   bool is_in(uintptr_t addr) const;
+  bool is_in_page_relaxed(const ZPage* page, zaddress addr) const;
 
-  // Threads
-  uint active_workers() const;
-  void set_active_workers(uint nworkers);
-  void threads_do(ThreadClosure* tc) const;
+  bool is_young(zaddress addr) const;
+  bool is_young(volatile zpointer* ptr) const;
 
-  // Reference processing
-  ReferenceDiscoverer* reference_discoverer();
-  void set_soft_reference_policy(bool clear);
+  bool is_old(zaddress addr) const;
+  bool is_old(volatile zpointer* ptr) const;
 
-  // Non-strong reference processing
-  void process_non_strong_references();
+  ZPage* page(zaddress addr) const;
+  ZPage* page(volatile zpointer* addr) const;
+
+  // Liveness
+  bool is_object_live(zaddress addr) const;
+  bool is_object_strongly_live(zaddress addr) const;
+  void keep_alive(oop obj);
+  void mark_flush_and_free(Thread* thread);
 
   // Page allocation
-  ZPage* alloc_page(uint8_t type, size_t size, ZAllocationFlags flags);
+  ZPage* alloc_page(ZPageType type, size_t size, ZAllocationFlags flags, ZPageAge age);
   void undo_alloc_page(ZPage* page);
-  void free_page(ZPage* page, bool reclaimed);
-  void free_pages(const ZArray<ZPage*>* pages, bool reclaimed);
+  void free_page(ZPage* page);
+  size_t free_empty_pages(const ZArray<ZPage*>* pages);
 
   // Object allocation
-  uintptr_t alloc_tlab(size_t size);
-  uintptr_t alloc_object(size_t size);
-  uintptr_t alloc_object_for_relocation(size_t size);
-  void undo_alloc_object_for_relocation(uintptr_t addr, size_t size);
-  bool has_alloc_stalled() const;
-  void check_out_of_memory();
-
-  // Marking
-  bool is_object_live(uintptr_t addr) const;
-  bool is_object_strongly_live(uintptr_t addr) const;
-  template <bool gc_thread, bool follow, bool finalizable, bool publish> void mark_object(uintptr_t addr);
-  void mark_start();
-  void mark(bool initial);
-  void mark_flush_and_free(Thread* thread);
-  bool mark_end();
-  void mark_free();
-  void keep_alive(oop obj);
-
-  // Relocation set
-  void select_relocation_set();
-  void reset_relocation_set();
-
-  // Relocation
-  void relocate_start();
-  uintptr_t relocate_object(uintptr_t addr);
-  uintptr_t remap_object(uintptr_t addr);
-  void relocate();
+  bool is_alloc_stalling() const;
+  bool is_alloc_stalling_for_old() const;
+  void handle_alloc_stalling_for_young();
+  void handle_alloc_stalling_for_old();
 
   // Continuations
-  bool is_allocating(uintptr_t addr) const;
+  bool is_allocating(zaddress addr) const;
 
   // Iteration
-  void object_iterate(ObjectClosure* cl, bool visit_weaks);
+  void object_iterate(ObjectClosure* object_cl, bool visit_weaks);
+  void object_and_field_iterate(ObjectClosure* object_cl, OopFieldClosure* field_cl, bool visit_weaks);
   ParallelObjectIteratorImpl* parallel_object_iterator(uint nworkers, bool visit_weaks);
-  void pages_do(ZPageClosure* cl);
+
+  void threads_do(ThreadClosure* tc) const;
 
   // Serviceability
   void serviceability_initialize();
-  GCMemoryManager* serviceability_cycle_memory_manager();
-  GCMemoryManager* serviceability_pause_memory_manager();
-  MemoryPool* serviceability_memory_pool();
+  GCMemoryManager* serviceability_cycle_memory_manager(bool minor);
+  GCMemoryManager* serviceability_pause_memory_manager(bool minor);
+  MemoryPool* serviceability_memory_pool(ZGenerationId id);
   ZServiceabilityCounters* serviceability_counters();
 
   // Printing
   void print_on(outputStream* st) const;
   void print_extended_on(outputStream* st) const;
   bool print_location(outputStream* st, uintptr_t addr) const;
+  bool print_location(outputStream* st, zaddress addr) const;
+  bool print_location(outputStream* st, zpointer ptr) const;
 
   // Verification
   bool is_oop(uintptr_t addr) const;
-  void verify();
 };
 
 #endif // SHARE_GC_Z_ZHEAP_HPP
diff --git a/src/hotspot/share/gc/z/zHeap.inline.hpp b/src/hotspot/share/gc/z/zHeap.inline.hpp
index 22f780a22cf..56c274b774a 100644
--- a/src/hotspot/share/gc/z/zHeap.inline.hpp
+++ b/src/hotspot/share/gc/z/zHeap.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,100 +28,70 @@
 
 #include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zForwardingTable.inline.hpp"
+#include "gc/z/zGenerationId.hpp"
 #include "gc/z/zMark.inline.hpp"
 #include "gc/z/zPage.inline.hpp"
 #include "gc/z/zPageTable.inline.hpp"
+#include "gc/z/zRemembered.inline.hpp"
 #include "utilities/debug.hpp"
 
 inline ZHeap* ZHeap::heap() {
-  assert(_heap != NULL, "Not initialized");
+  assert(_heap != nullptr, "Not initialized");
   return _heap;
 }
 
-inline ReferenceDiscoverer* ZHeap::reference_discoverer() {
-  return &_reference_processor;
+inline bool ZHeap::is_young(zaddress addr) const {
+  return page(addr)->is_young();
 }
 
-inline bool ZHeap::is_object_live(uintptr_t addr) const {
-  ZPage* page = _page_table.get(addr);
+inline bool ZHeap::is_young(volatile zpointer* ptr) const {
+  return page(ptr)->is_young();
+}
+
+inline bool ZHeap::is_old(zaddress addr) const {
+  return !is_young(addr);
+}
+
+inline bool ZHeap::is_old(volatile zpointer* ptr) const {
+  return !is_young(ptr);
+}
+
+inline ZPage* ZHeap::page(zaddress addr) const {
+  return _page_table.get(addr);
+}
+
+inline ZPage* ZHeap::page(volatile zpointer* ptr) const {
+  return _page_table.get(ptr);
+}
+
+inline bool ZHeap::is_object_live(zaddress addr) const {
+  const ZPage* const page = _page_table.get(addr);
   return page->is_object_live(addr);
 }
 
-inline bool ZHeap::is_object_strongly_live(uintptr_t addr) const {
-  ZPage* page = _page_table.get(addr);
+inline bool ZHeap::is_object_strongly_live(zaddress addr) const {
+  const ZPage* const page = _page_table.get(addr);
   return page->is_object_strongly_live(addr);
 }
 
-template <bool gc_thread, bool follow, bool finalizable, bool publish>
-inline void ZHeap::mark_object(uintptr_t addr) {
-  assert(ZGlobalPhase == ZPhaseMark, "Mark not allowed");
-  _mark.mark_object<gc_thread, follow, finalizable, publish>(addr);
+inline bool ZHeap::is_alloc_stalling() const {
+  return _page_allocator.is_alloc_stalling();
 }
 
-inline uintptr_t ZHeap::alloc_tlab(size_t size) {
-  guarantee(size <= max_tlab_size(), "TLAB too large");
-  return _object_allocator.alloc_object(size);
+inline bool ZHeap::is_alloc_stalling_for_old() const {
+  return _page_allocator.is_alloc_stalling_for_old();
 }
 
-inline uintptr_t ZHeap::alloc_object(size_t size) {
-  uintptr_t addr = _object_allocator.alloc_object(size);
-  assert(ZAddress::is_good_or_null(addr), "Bad address");
-
-  if (addr == 0) {
-    out_of_memory();
-  }
-
-  return addr;
+inline void ZHeap::handle_alloc_stalling_for_young() {
+  _page_allocator.handle_alloc_stalling_for_young();
 }
 
-inline uintptr_t ZHeap::alloc_object_for_relocation(size_t size) {
-  const uintptr_t addr = _object_allocator.alloc_object_for_relocation(&_page_table, size);
-  assert(ZAddress::is_good_or_null(addr), "Bad address");
-  return addr;
-}
-
-inline void ZHeap::undo_alloc_object_for_relocation(uintptr_t addr, size_t size) {
-  ZPage* const page = _page_table.get(addr);
-  _object_allocator.undo_alloc_object_for_relocation(page, addr, size);
-}
-
-inline uintptr_t ZHeap::relocate_object(uintptr_t addr) {
-  assert(ZGlobalPhase == ZPhaseRelocate, "Relocate not allowed");
-
-  ZForwarding* const forwarding = _forwarding_table.get(addr);
-  if (forwarding == NULL) {
-    // Not forwarding
-    return ZAddress::good(addr);
-  }
-
-  // Relocate object
-  return _relocate.relocate_object(forwarding, ZAddress::good(addr));
-}
-
-inline uintptr_t ZHeap::remap_object(uintptr_t addr) {
-  assert(ZGlobalPhase == ZPhaseMark ||
-         ZGlobalPhase == ZPhaseMarkCompleted, "Forward not allowed");
-
-  ZForwarding* const forwarding = _forwarding_table.get(addr);
-  if (forwarding == NULL) {
-    // Not forwarding
-    return ZAddress::good(addr);
-  }
-
-  // Forward object
-  return _relocate.forward_object(forwarding, ZAddress::good(addr));
-}
-
-inline bool ZHeap::has_alloc_stalled() const {
-  return _page_allocator.has_alloc_stalled();
-}
-
-inline void ZHeap::check_out_of_memory() {
-  _page_allocator.check_out_of_memory();
+inline void ZHeap::handle_alloc_stalling_for_old() {
+  _page_allocator.handle_alloc_stalling_for_old();
 }
 
 inline bool ZHeap::is_oop(uintptr_t addr) const {
-  return ZAddress::is_good(addr) && is_object_aligned(addr) && is_in(addr);
+  return is_in(addr);
 }
 
 #endif // SHARE_GC_Z_ZHEAP_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zHeapIterator.cpp b/src/hotspot/share/gc/z/zHeapIterator.cpp
index f2eb79c0d55..a66ae55798e 100644
--- a/src/hotspot/share/gc/z/zHeapIterator.cpp
+++ b/src/hotspot/share/gc/z/zHeapIterator.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,18 +22,21 @@
  */
 
 #include "precompiled.hpp"
+#include "classfile/classLoaderData.hpp"
 #include "classfile/classLoaderDataGraph.hpp"
+#include "gc/shared/barrierSet.hpp"
 #include "gc/shared/barrierSetNMethod.hpp"
 #include "gc/shared/gc_globals.hpp"
 #include "gc/shared/taskqueue.inline.hpp"
 #include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zCollectedHeap.hpp"
+#include "gc/z/zGenerationId.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zGranuleMap.inline.hpp"
+#include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zHeapIterator.hpp"
 #include "gc/z/zLock.inline.hpp"
 #include "gc/z/zNMethod.hpp"
-#include "gc/z/zOop.inline.hpp"
 #include "memory/iterator.inline.hpp"
 #include "utilities/bitMap.inline.hpp"
 
@@ -56,17 +59,31 @@ private:
   ZHeapIteratorQueue* const      _queue;
   ZHeapIteratorArrayQueue* const _array_queue;
   const uint                     _worker_id;
-  ZStatTimerDisable              _timer_disable;
+  ObjectClosure*                 _object_cl;
+  OopFieldClosure*               _field_cl;
 
 public:
-  ZHeapIteratorContext(ZHeapIterator* iter, uint worker_id) :
+  ZHeapIteratorContext(ZHeapIterator* iter, ObjectClosure* object_cl, OopFieldClosure* field_cl, uint worker_id) :
       _iter(iter),
       _queue(_iter->_queues.queue(worker_id)),
       _array_queue(_iter->_array_queues.queue(worker_id)),
-      _worker_id(worker_id) {}
+      _worker_id(worker_id),
+      _object_cl(object_cl),
+      _field_cl(field_cl) {}
+
+  void visit_field(oop base, oop* p) const {
+    if (_field_cl != nullptr) {
+      _field_cl->do_field(base, p);
+    }
+  }
+
+  void visit_object(oop obj) const {
+    _object_cl->do_object(obj);
+  }
 
   void mark_and_push(oop obj) const {
     if (_iter->mark_object(obj)) {
+      visit_object(obj);
       _queue->push(obj);
     }
   }
@@ -97,7 +114,7 @@ public:
 };
 
 template <bool Weak>
-class ZHeapIteratorRootOopClosure : public OopClosure {
+class ZHeapIteratorColoredRootOopClosure : public OopClosure {
 private:
   const ZHeapIteratorContext& _context;
 
@@ -110,10 +127,36 @@ private:
   }
 
 public:
-  ZHeapIteratorRootOopClosure(const ZHeapIteratorContext& context) :
+  ZHeapIteratorColoredRootOopClosure(const ZHeapIteratorContext& context) :
       _context(context) {}
 
   virtual void do_oop(oop* p) {
+    _context.visit_field(nullptr, p);
+    const oop obj = load_oop(p);
+    _context.mark_and_push(obj);
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+};
+
+class ZHeapIteratorUncoloredRootOopClosure : public OopClosure {
+private:
+  const ZHeapIteratorContext& _context;
+
+  oop load_oop(oop* p) {
+    const oop o = Atomic::load(p);
+    assert_is_valid(to_zaddress(o));
+    return RawAccess<>::oop_load(p);
+  }
+
+public:
+  ZHeapIteratorUncoloredRootOopClosure(const ZHeapIteratorContext& context) :
+      _context(context) {}
+
+  virtual void do_oop(oop* p) {
+    _context.visit_field(nullptr, p);
     const oop obj = load_oop(p);
     _context.mark_and_push(obj);
   }
@@ -150,6 +193,7 @@ public:
   }
 
   virtual void do_oop(oop* p) {
+    _context.visit_field(_base, p);
     const oop obj = load_oop(p);
     _context.mark_and_push(obj);
   }
@@ -198,13 +242,13 @@ public:
 
 ZHeapIterator::ZHeapIterator(uint nworkers, bool visit_weaks) :
     _visit_weaks(visit_weaks),
-    _timer_disable(),
     _bitmaps(ZAddressOffsetMax),
     _bitmaps_lock(),
     _queues(nworkers),
     _array_queues(nworkers),
-    _roots(ClassLoaderData::_claim_other),
-    _weak_roots(),
+    _roots_colored(ZGenerationIdOptional::none),
+    _roots_uncolored(ZGenerationIdOptional::none),
+    _roots_weak_colored(ZGenerationIdOptional::none),
     _terminator(nworkers, &_queues) {
 
   // Create queues
@@ -246,19 +290,19 @@ static size_t object_index_max() {
 }
 
 static size_t object_index(oop obj) {
-  const uintptr_t addr = ZOop::to_address(obj);
-  const uintptr_t offset = ZAddress::offset(addr);
+  const zaddress addr = to_zaddress(obj);
+  const zoffset offset = ZAddress::offset(addr);
   const uintptr_t mask = ZGranuleSize - 1;
-  return (offset & mask) >> ZObjectAlignmentSmallShift;
+  return (untype(offset) & mask) >> ZObjectAlignmentSmallShift;
 }
 
 ZHeapIteratorBitMap* ZHeapIterator::object_bitmap(oop obj) {
-  const uintptr_t offset = ZAddress::offset(ZOop::to_address(obj));
+  const zoffset offset = ZAddress::offset(to_zaddress(obj));
   ZHeapIteratorBitMap* bitmap = _bitmaps.get_acquire(offset);
-  if (bitmap == NULL) {
+  if (bitmap == nullptr) {
     ZLocker<ZLock> locker(&_bitmaps_lock);
     bitmap = _bitmaps.get(offset);
-    if (bitmap == NULL) {
+    if (bitmap == nullptr) {
       // Install new bitmap
       bitmap = new ZHeapIteratorBitMap(object_index_max());
       _bitmaps.release_put(offset, bitmap);
@@ -269,7 +313,7 @@ ZHeapIteratorBitMap* ZHeapIterator::object_bitmap(oop obj) {
 }
 
 bool ZHeapIterator::mark_object(oop obj) {
-  if (obj == NULL) {
+  if (obj == nullptr) {
     return false;
   }
 
@@ -278,7 +322,7 @@ bool ZHeapIterator::mark_object(oop obj) {
   return bitmap->try_set_bit(index);
 }
 
-typedef ClaimingCLDToOopClosure<ClassLoaderData::_claim_other> ZHeapIteratorCLDCLosure;
+typedef ClaimingCLDToOopClosure<ClassLoaderData::_claim_other> ZHeapIteratorCLDClosure;
 
 class ZHeapIteratorNMethodClosure : public NMethodClosure {
 private:
@@ -317,20 +361,26 @@ public:
 };
 
 void ZHeapIterator::push_strong_roots(const ZHeapIteratorContext& context) {
-  ZHeapIteratorRootOopClosure<false /* Weak */> cl(context);
-  ZHeapIteratorCLDCLosure cld_cl(&cl);
-  ZHeapIteratorNMethodClosure nm_cl(&cl);
-  ZHeapIteratorThreadClosure thread_cl(&cl, &nm_cl);
+  {
+    ZHeapIteratorColoredRootOopClosure<false /* Weak */> cl(context);
+    ZHeapIteratorCLDClosure cld_cl(&cl);
 
-  _roots.apply(&cl,
-               &cld_cl,
-               &thread_cl,
-               &nm_cl);
+    _roots_colored.apply(&cl,
+                         &cld_cl);
+  }
+
+  {
+    ZHeapIteratorUncoloredRootOopClosure cl(context);
+    ZHeapIteratorNMethodClosure nm_cl(&cl);
+    ZHeapIteratorThreadClosure thread_cl(&cl, &nm_cl);
+    _roots_uncolored.apply(&thread_cl,
+                           &nm_cl);
+  }
 }
 
 void ZHeapIterator::push_weak_roots(const ZHeapIteratorContext& context) {
-  ZHeapIteratorRootOopClosure<true  /* Weak */> cl(context);
-  _weak_roots.apply(&cl);
+  ZHeapIteratorColoredRootOopClosure<true  /* Weak */> cl(context);
+  _roots_weak_colored.apply(&cl);
 }
 
 template <bool VisitWeaks>
@@ -344,7 +394,7 @@ void ZHeapIterator::push_roots(const ZHeapIteratorContext& context) {
 template <bool VisitReferents>
 void ZHeapIterator::follow_object(const ZHeapIteratorContext& context, oop obj) {
   ZHeapIteratorOopClosure<VisitReferents> cl(context, obj);
-  obj->oop_iterate(&cl);
+  ZIterator::oop_iterate(obj, &cl);
 }
 
 void ZHeapIterator::follow_array(const ZHeapIteratorContext& context, oop obj) {
@@ -370,14 +420,11 @@ void ZHeapIterator::follow_array_chunk(const ZHeapIteratorContext& context, cons
 
   // Follow array chunk
   ZHeapIteratorOopClosure<false /* VisitReferents */> cl(context, obj);
-  obj->oop_iterate_range(&cl, start, end);
+  ZIterator::oop_iterate_range(obj, &cl, start, end);
 }
 
 template <bool VisitWeaks>
-void ZHeapIterator::visit_and_follow(const ZHeapIteratorContext& context, ObjectClosure* cl, oop obj) {
-  // Visit
-  cl->do_object(obj);
-
+void ZHeapIterator::follow(const ZHeapIteratorContext& context, oop obj) {
   // Follow
   if (obj->is_objArray()) {
     follow_array(context, obj);
@@ -387,13 +434,13 @@ void ZHeapIterator::visit_and_follow(const ZHeapIteratorContext& context, Object
 }
 
 template <bool VisitWeaks>
-void ZHeapIterator::drain(const ZHeapIteratorContext& context, ObjectClosure* cl) {
+void ZHeapIterator::drain(const ZHeapIteratorContext& context) {
   ObjArrayTask array;
   oop obj;
 
   do {
     while (context.pop(obj)) {
-      visit_and_follow<VisitWeaks>(context, cl, obj);
+      follow<VisitWeaks>(context, obj);
     }
 
     if (context.pop_array(array)) {
@@ -403,37 +450,47 @@ void ZHeapIterator::drain(const ZHeapIteratorContext& context, ObjectClosure* cl
 }
 
 template <bool VisitWeaks>
-void ZHeapIterator::steal(const ZHeapIteratorContext& context, ObjectClosure* cl) {
+void ZHeapIterator::steal(const ZHeapIteratorContext& context) {
   ObjArrayTask array;
   oop obj;
 
   if (context.steal_array(array)) {
     follow_array_chunk(context, array);
   } else if (context.steal(obj)) {
-    visit_and_follow<VisitWeaks>(context, cl, obj);
+    follow<VisitWeaks>(context, obj);
   }
 }
 
 template <bool VisitWeaks>
-void ZHeapIterator::drain_and_steal(const ZHeapIteratorContext& context, ObjectClosure* cl) {
+void ZHeapIterator::drain_and_steal(const ZHeapIteratorContext& context) {
   do {
-    drain<VisitWeaks>(context, cl);
-    steal<VisitWeaks>(context, cl);
+    drain<VisitWeaks>(context);
+    steal<VisitWeaks>(context);
   } while (!context.is_drained() || !_terminator.offer_termination());
 }
 
 template <bool VisitWeaks>
-void ZHeapIterator::object_iterate_inner(const ZHeapIteratorContext& context, ObjectClosure* object_cl) {
+void ZHeapIterator::object_iterate_inner(const ZHeapIteratorContext& context) {
   push_roots<VisitWeaks>(context);
-  drain_and_steal<VisitWeaks>(context, object_cl);
+  drain_and_steal<VisitWeaks>(context);
 }
 
-void ZHeapIterator::object_iterate(ObjectClosure* cl, uint worker_id) {
-  ZHeapIteratorContext context(this, worker_id);
+void ZHeapIterator::object_iterate(ObjectClosure* object_cl, uint worker_id) {
+  const ZHeapIteratorContext context(this, object_cl, nullptr /* field_cl */, worker_id);
 
   if (_visit_weaks) {
-    object_iterate_inner<true /* VisitWeaks */>(context, cl);
+    object_iterate_inner<true /* VisitWeaks */>(context);
   } else {
-    object_iterate_inner<false /* VisitWeaks */>(context, cl);
+    object_iterate_inner<false /* VisitWeaks */>(context);
+  }
+}
+
+void ZHeapIterator::object_and_field_iterate(ObjectClosure* object_cl, OopFieldClosure* field_cl, uint worker_id) {
+  const ZHeapIteratorContext context(this, object_cl, field_cl, worker_id);
+
+  if (_visit_weaks) {
+    object_iterate_inner<true /* VisitWeaks */>(context);
+  } else {
+    object_iterate_inner<false /* VisitWeaks */>(context);
   }
 }
diff --git a/src/hotspot/share/gc/z/zHeapIterator.hpp b/src/hotspot/share/gc/z/zHeapIterator.hpp
index 5c3a82d8bb7..27d9e2f0df1 100644
--- a/src/hotspot/share/gc/z/zHeapIterator.hpp
+++ b/src/hotspot/share/gc/z/zHeapIterator.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -36,7 +36,7 @@ class ZHeapIteratorBitMap;
 class ZHeapIteratorContext;
 
 using ZHeapIteratorBitMaps = ZGranuleMap<ZHeapIteratorBitMap*>;
-using ZHeapIteratorBitMapsIterator = ZGranuleMapIterator<ZHeapIteratorBitMap*>;
+using ZHeapIteratorBitMapsIterator = ZGranuleMapIterator<ZHeapIteratorBitMap*, false /* Parallel */>;
 using ZHeapIteratorQueue = OverflowTaskQueue<oop, mtGC>;
 using ZHeapIteratorQueues = GenericTaskQueueSet<ZHeapIteratorQueue, mtGC>;
 using ZHeapIteratorArrayQueue = OverflowTaskQueue<ObjArrayTask, mtGC>;
@@ -44,17 +44,18 @@ using ZHeapIteratorArrayQueues = GenericTaskQueueSet<ZHeapIteratorArrayQueue, mt
 
 class ZHeapIterator : public ParallelObjectIteratorImpl {
   friend class ZHeapIteratorContext;
+  friend class ZHeapIteratorRootUncoloredOopClosure;
 
 private:
-  const bool               _visit_weaks;
-  ZStatTimerDisable        _timer_disable;
-  ZHeapIteratorBitMaps     _bitmaps;
-  ZLock                    _bitmaps_lock;
-  ZHeapIteratorQueues      _queues;
-  ZHeapIteratorArrayQueues _array_queues;
-  ZRootsIterator           _roots;
-  ZWeakRootsIterator       _weak_roots;
-  TaskTerminator           _terminator;
+  const bool                    _visit_weaks;
+  ZHeapIteratorBitMaps          _bitmaps;
+  ZLock                         _bitmaps_lock;
+  ZHeapIteratorQueues           _queues;
+  ZHeapIteratorArrayQueues      _array_queues;
+  ZRootsIteratorStrongColored   _roots_colored;
+  ZRootsIteratorStrongUncolored _roots_uncolored;
+  ZRootsIteratorWeakColored     _roots_weak_colored;
+  TaskTerminator                _terminator;
 
   ZHeapIteratorBitMap* object_bitmap(oop obj);
 
@@ -73,25 +74,26 @@ private:
   void follow_array_chunk(const ZHeapIteratorContext& context, const ObjArrayTask& array);
 
   template <bool VisitWeaks>
-  void visit_and_follow(const ZHeapIteratorContext& context, ObjectClosure* cl, oop obj);
+  void follow(const ZHeapIteratorContext& context, oop obj);
 
   template <bool VisitWeaks>
-  void drain(const ZHeapIteratorContext& context, ObjectClosure* cl);
+  void drain(const ZHeapIteratorContext& context);
 
   template <bool VisitWeaks>
-  void steal(const ZHeapIteratorContext& context, ObjectClosure* cl);
+  void steal(const ZHeapIteratorContext& context);
 
   template <bool VisitWeaks>
-  void drain_and_steal(const ZHeapIteratorContext& context, ObjectClosure* cl);
+  void drain_and_steal(const ZHeapIteratorContext& context);
 
   template <bool VisitWeaks>
-  void object_iterate_inner(const ZHeapIteratorContext& context, ObjectClosure* cl);
+  void object_iterate_inner(const ZHeapIteratorContext& context);
 
 public:
   ZHeapIterator(uint nworkers, bool visit_weaks);
   virtual ~ZHeapIterator();
 
-  virtual void object_iterate(ObjectClosure* cl, uint worker_id);
+  virtual void object_iterate(ObjectClosure* object_cl, uint worker_id);
+  void object_and_field_iterate(ObjectClosure* object_cl, OopFieldClosure* field_cl, uint worker_id);
 };
 
 #endif // SHARE_GC_Z_ZHEAPITERATOR_HPP
diff --git a/src/hotspot/share/gc/z/zHeuristics.cpp b/src/hotspot/share/gc/z/zHeuristics.cpp
index a999d2ab4aa..bcd9dd84405 100644
--- a/src/hotspot/share/gc/z/zHeuristics.cpp
+++ b/src/hotspot/share/gc/z/zHeuristics.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,8 +22,8 @@
  */
 
 #include "precompiled.hpp"
-#include "gc/shared/gcLogPrecious.hpp"
 #include "gc/shared/gc_globals.hpp"
+#include "gc/shared/gcLogPrecious.hpp"
 #include "gc/z/zCPU.inline.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zHeuristics.hpp"
@@ -56,15 +56,14 @@ void ZHeuristics::set_medium_page_size() {
 size_t ZHeuristics::relocation_headroom() {
   // Calculate headroom needed to avoid in-place relocation. Each worker will try
   // to allocate a small page, and all workers will share a single medium page.
-  const uint nworkers = UseDynamicNumberOfGCThreads ? ConcGCThreads : MAX2(ConcGCThreads, ParallelGCThreads);
-  return (nworkers * ZPageSizeSmall) + ZPageSizeMedium;
+  return (ConcGCThreads * ZPageSizeSmall) + ZPageSizeMedium;
 }
 
 bool ZHeuristics::use_per_cpu_shared_small_pages() {
   // Use per-CPU shared small pages only if these pages occupy at most 3.125%
   // of the max heap size. Otherwise fall back to using a single shared small
   // page. This is useful when using small heaps on large machines.
-  const size_t per_cpu_share = (MaxHeapSize * 0.03125) / ZCPU::count();
+  const size_t per_cpu_share = significant_heap_overhead() / ZCPU::count();
   return per_cpu_share >= ZPageSizeSmall;
 }
 
@@ -73,8 +72,7 @@ static uint nworkers_based_on_ncpus(double cpu_share_in_percent) {
 }
 
 static uint nworkers_based_on_heap_size(double heap_share_in_percent) {
-  const int nworkers = (MaxHeapSize * (heap_share_in_percent / 100.0)) / ZPageSizeSmall;
-  return MAX2(nworkers, 1);
+  return (MaxHeapSize * (heap_share_in_percent / 100.0)) / ZPageSizeSmall;
 }
 
 static uint nworkers(double cpu_share_in_percent) {
@@ -90,15 +88,22 @@ uint ZHeuristics::nparallel_workers() {
   // close to the number of processors tends to lead to over-provisioning and
   // scheduling latency issues. Using 60% of the active processors appears to
   // be a fairly good balance.
-  return nworkers(60.0);
+  return MAX2(nworkers(60.0), 1u);
 }
 
 uint ZHeuristics::nconcurrent_workers() {
   // The number of concurrent threads we would like to use heavily depends
   // on the type of workload we are running. Using too many threads will have
   // a negative impact on the application throughput, while using too few
-  // threads will prolong the GC-cycle and we then risk being out-run by the
-  // application. When in dynamic mode, use up to 25% of the active processors.
-  //  When in non-dynamic mode, use 12.5% of the active processors.
-  return nworkers(UseDynamicNumberOfGCThreads ? 25.0 : 12.5);
+  // threads will prolong the GC cycle and we then risk being out-run by the
+  // application.
+  return MAX2(nworkers(25.0), 1u);
+}
+
+size_t ZHeuristics::significant_heap_overhead() {
+  return MaxHeapSize * ZFragmentationLimit;
+}
+
+size_t ZHeuristics::significant_young_overhead() {
+  return MaxHeapSize * ZYoungCompactionLimit;
 }
diff --git a/src/hotspot/share/gc/z/zHeuristics.hpp b/src/hotspot/share/gc/z/zHeuristics.hpp
index 362fd775f0f..0e2d851216c 100644
--- a/src/hotspot/share/gc/z/zHeuristics.hpp
+++ b/src/hotspot/share/gc/z/zHeuristics.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
 #define SHARE_GC_Z_ZHEURISTICS_HPP
 
 #include "memory/allStatic.hpp"
+#include "utilities/globalDefinitions.hpp"
 
 class ZHeuristics : public AllStatic {
 public:
@@ -36,6 +37,9 @@ public:
 
   static uint nparallel_workers();
   static uint nconcurrent_workers();
+
+  static size_t significant_heap_overhead();
+  static size_t significant_young_overhead();
 };
 
 #endif // SHARE_GC_Z_ZHEURISTICS_HPP
diff --git a/src/hotspot/share/gc/z/zIndexDistributor.hpp b/src/hotspot/share/gc/z/zIndexDistributor.hpp
new file mode 100644
index 00000000000..94f146176d6
--- /dev/null
+++ b/src/hotspot/share/gc/z/zIndexDistributor.hpp
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZINDEXDISTRIBUTOR_HPP
+#define SHARE_GC_Z_ZINDEXDISTRIBUTOR_HPP
+
+class ZIndexDistributor {
+private:
+  void* _strategy;
+
+  template <typename Strategy>
+  Strategy* strategy();
+
+  static void* create_strategy(int count);
+
+public:
+  ZIndexDistributor(int count);
+  ~ZIndexDistributor();
+
+  template <typename Function>
+  void do_indices(Function function);
+};
+
+#endif // SHARE_GC_Z_ZINDEXDISTRIBUTOR_HPP
diff --git a/src/hotspot/share/gc/z/zIndexDistributor.inline.hpp b/src/hotspot/share/gc/z/zIndexDistributor.inline.hpp
new file mode 100644
index 00000000000..6a54ea3876c
--- /dev/null
+++ b/src/hotspot/share/gc/z/zIndexDistributor.inline.hpp
@@ -0,0 +1,331 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZINDEXDISTRIBUTOR_INLINE_HPP
+#define SHARE_GC_Z_ZINDEXDISTRIBUTOR_INLINE_HPP
+
+#include "gc/z/zIndexDistributor.hpp"
+
+#include "gc/shared/gc_globals.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/os.hpp"
+#include "runtime/thread.hpp"
+#include "utilities/align.hpp"
+
+class ZIndexDistributorStriped : public CHeapObj<mtGC> {
+  static const int MemSize = 4096;
+
+  const int _max_index;
+  // For claiming a stripe
+  volatile int _claim_stripe;
+  // For claiming inside a stripe
+  char _mem[MemSize + ZCacheLineSize];
+
+  int claim_stripe() {
+    return Atomic::fetch_and_add(&_claim_stripe, 1, memory_order_relaxed);
+  }
+
+  volatile int* claim_addr(int index) {
+    return (volatile int*)(align_up(_mem, ZCacheLineSize) + index * ZCacheLineSize);
+  }
+
+public:
+  ZIndexDistributorStriped(int max_index) :
+      _max_index(max_index),
+      _claim_stripe(0),
+      _mem() {
+    memset(_mem, 0, MemSize + ZCacheLineSize);
+  }
+
+  template <typename Function>
+  void do_indices(Function function) {
+    const int count = MemSize / ZCacheLineSize;
+    const int stripe_max = _max_index / count;
+
+    // Use claiming
+    for (int i; (i = claim_stripe()) < count;) {
+      for (int index; (index = Atomic::fetch_and_add(claim_addr(i), 1, memory_order_relaxed)) < stripe_max;) {
+        if (!function(i * stripe_max + index)) {
+          return;
+        }
+      }
+    }
+
+    // Use stealing
+    for (int i = 0; i < count; i++) {
+      for (int index; (index = Atomic::fetch_and_add(claim_addr(i), 1, memory_order_relaxed)) < stripe_max;) {
+        if (!function(i * stripe_max + index)) {
+          return;
+        }
+      }
+    }
+  }
+};
+
+class ZIndexDistributorClaimTree : public CHeapObj<mtGC> {
+  friend class ZIndexDistributorTest;
+
+private:
+  // The N - 1 levels are used to claim a segment in the
+  // next level the Nth level claims an index.
+  static constexpr int N = 4;
+  static constexpr int ClaimLevels = N - 1;
+
+  // Describes the how the number of indices increases when going up from the given level
+  static constexpr int level_multiplier(int level) {
+    assert(level < ClaimLevels, "Must be");
+    constexpr int array[ClaimLevels]{16, 16, 16};
+    return array[level];
+  }
+
+  // Number of indices in one segment at the last level
+  const int     _last_level_segment_size_shift;
+
+  // For deallocation
+  char*         _malloced;
+
+  // Contains the tree of claim variables
+  volatile int* _claim_array;
+
+  // Claim index functions
+
+  // Number of claim entries at the given level
+  static constexpr int claim_level_size(int level) {
+    if (level == 0) {
+      return 1;
+    }
+
+    return level_multiplier(level - 1) * claim_level_size(level - 1);
+  }
+
+  // The index the next level starts at
+  static constexpr int claim_level_end_index(int level) {
+    if (level == 0) {
+
+      // First level uses padding
+      return ZCacheLineSize / sizeof(int);
+    }
+
+    return claim_level_size(level) + claim_level_end_index(level - 1);
+  }
+
+  static constexpr int claim_level_start_index(int level) {
+    return claim_level_end_index(level - 1);
+  }
+
+  // Total size used to hold all claim variables
+  static size_t claim_variables_size() {
+    return sizeof(int) * claim_level_end_index(ClaimLevels);
+  }
+
+  // Returns the index of the start of the current segment of the current level
+  static constexpr int claim_level_index_accumulate(int* indices, int level, int acc = 1) {
+    if (level == 0) {
+      return acc * indices[level];
+    }
+
+    return acc * indices[level] + claim_level_index_accumulate(indices, level - 1, acc * level_multiplier(level));
+  }
+
+  static constexpr int claim_level_index(int* indices, int level) {
+    assert(level > 0, "Must be");
+
+    // The claim index for the current level is found in the previous levels
+    return claim_level_index_accumulate(indices, level - 1);
+  }
+
+  static constexpr int claim_index(int* indices, int level) {
+    if (level == 0) {
+      return 0;
+    }
+
+    return claim_level_start_index(level) + claim_level_index(indices, level);
+  }
+
+  // Claim functions
+
+  int claim(int index) {
+    return Atomic::fetch_and_add(&_claim_array[index], 1, memory_order_relaxed);
+  }
+
+  int claim_at(int* indices, int level) {
+    const int index = claim_index(indices, level);
+    const int value = claim(index);
+#if 0
+    if      (level == 0) { tty->print_cr("Claim at: %d index: %d got: %d",             indices[0], index, value); }
+    else if (level == 1) { tty->print_cr("Claim at: %d %d index: %d got: %d",          indices[0], indices[1], index, value); }
+    else if (level == 2) { tty->print_cr("Claim at: %d %d %d index: %d got: %d",       indices[0], indices[1], indices[2], index, value); }
+    else if (level == 3) { tty->print_cr("Claim at: %d %d %d %d index: %d got: %d",    indices[0], indices[1], indices[2], indices[3], index, value); }
+    else if (level == 4) { tty->print_cr("Claim at: %d %d %d %d %d index: %d got: %d", indices[0], indices[1], indices[2], indices[3], indices[4], index, value); }
+#endif
+    return value;
+  }
+
+  template <typename Function>
+  void claim_and_do(Function function, int* indices, int level) {
+    if (level < N) {
+      // Visit ClaimLevels and the last level
+      const int ci = claim_index(indices, level);
+      for (indices[level] = 0; (indices[level] = claim(ci)) < level_segment_size(level);) {
+        claim_and_do(function, indices, level + 1);
+      }
+      return;
+    }
+
+    doit(function, indices);
+  }
+
+  template <typename Function>
+  void steal_and_do(Function function, int* indices, int level) {
+    for (indices[level] = 0; indices[level] < level_segment_size(level); indices[level]++) {
+      const int next_level = level + 1;
+      // First try to claim at next level
+      claim_and_do(function, indices, next_level);
+      // Then steal at next level
+      if (next_level < ClaimLevels) {
+        steal_and_do(function, indices, next_level);
+      }
+    }
+  }
+
+  // Functions to claimed values to an index
+
+  static constexpr int levels_size(int level) {
+    if (level == 0) {
+      return level_multiplier(0);
+    }
+
+    return level_multiplier(level) * levels_size(level - 1);
+  }
+
+  static int constexpr level_to_last_level_count_coverage(int level) {
+    return levels_size(ClaimLevels - 1) / levels_size(level);
+  }
+
+  static int constexpr calculate_last_level_count(int* indices, int level = 0) {
+    if (level == N - 1) {
+      return 0;
+    }
+
+    return indices[level] * level_to_last_level_count_coverage(level) + calculate_last_level_count(indices, level + 1);
+  }
+
+  int calculate_index(int* indices) {
+    const int segment_start = calculate_last_level_count(indices) << _last_level_segment_size_shift;
+    return segment_start + indices[N - 1];
+  }
+
+  int level_segment_size(int level) {
+    if (level == ClaimLevels) {
+      return 1 << _last_level_segment_size_shift;
+    }
+
+    return level_multiplier(level);
+  }
+
+  template <typename Function>
+  void doit(Function function, int* indices) {
+    //const int index = first_level * second_level_max * _third_level_max + second_level * _third_level_max + third_level;
+    const int index = calculate_index(indices);
+
+#if 0
+    tty->print_cr("doit Thread: " PTR_FORMAT ": %d %d %d %d => %d",
+        p2i(Thread::current()),
+        indices[0], indices[1], indices[2], indices[3], index);
+#endif
+
+    function(index);
+  }
+
+  static int last_level_segment_size_shift(int count) {
+    const int last_level_size = count / levels_size(ClaimLevels - 1);
+    assert(levels_size(ClaimLevels - 1) * last_level_size == count, "Not exactly divisible");
+
+    return log2i_exact(last_level_size);
+  }
+
+public:
+  ZIndexDistributorClaimTree(int count) :
+      _last_level_segment_size_shift(last_level_segment_size_shift(count)),
+      _malloced((char*)os::malloc(claim_variables_size() + os::vm_page_size(), mtGC)),
+      _claim_array((volatile int*)align_up(_malloced, os::vm_page_size())) {
+
+    assert((levels_size(ClaimLevels - 1) << _last_level_segment_size_shift) == count, "Incorrectly setup");
+
+#if 0
+    tty->print_cr("ZIndexDistributorClaimTree count: %d byte size: " SIZE_FORMAT, count, claim_variables_size() + os::vm_page_size());
+#endif
+
+    memset(_malloced, 0, claim_variables_size() + os::vm_page_size());
+  }
+
+  ~ZIndexDistributorClaimTree() {
+    os::free(_malloced);
+  }
+
+  template <typename Function>
+  void do_indices(Function function) {
+    int indices[N];
+    claim_and_do(function, indices, 0 /* level */);
+    steal_and_do(function, indices, 0 /* level */);
+  }
+};
+
+// Using dynamically allocated objects just to be able to evaluate
+// different strategies. Revert when one has been choosen.
+
+inline void* ZIndexDistributor::create_strategy(int count) {
+  switch (ZIndexDistributorStrategy) {
+  case 0: return new ZIndexDistributorClaimTree(count);
+  case 1: return new ZIndexDistributorStriped(count);
+  default: fatal("Unknown ZIndexDistributorStrategy"); return nullptr;
+  };
+}
+
+inline ZIndexDistributor::ZIndexDistributor(int count) :
+    _strategy(create_strategy(count)) {}
+
+inline ZIndexDistributor::~ZIndexDistributor() {
+  switch (ZIndexDistributorStrategy) {
+  case 0: delete static_cast<ZIndexDistributorClaimTree*>(_strategy); break;
+  case 1: delete static_cast<ZIndexDistributorStriped*>(_strategy); break;
+  default: fatal("Unknown ZIndexDistributorStrategy"); break;
+  };
+}
+
+template <typename Strategy>
+inline Strategy* ZIndexDistributor::strategy() {
+  return static_cast<Strategy*>(_strategy);
+}
+
+template <typename Function>
+inline void ZIndexDistributor::do_indices(Function function) {
+  switch (ZIndexDistributorStrategy) {
+  case 0: strategy<ZIndexDistributorClaimTree>()->do_indices(function); break;
+  case 1: strategy<ZIndexDistributorStriped>()->do_indices(function); break;
+  default: fatal("Unknown ZIndexDistributorStrategy");
+  };
+}
+
+#endif // SHARE_GC_Z_ZINDEXDISTRIBUTOR_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zInitialize.cpp b/src/hotspot/share/gc/z/zInitialize.cpp
index 8bc64795315..0c0dc6e87a6 100644
--- a/src/hotspot/share/gc/z/zInitialize.cpp
+++ b/src/hotspot/share/gc/z/zInitialize.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,10 +25,14 @@
 #include "gc/z/zAddress.hpp"
 #include "gc/z/zBarrierSet.hpp"
 #include "gc/z/zCPU.hpp"
+#include "gc/z/zDriver.hpp"
+#include "gc/z/zGCIdPrinter.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zHeuristics.hpp"
 #include "gc/z/zInitialize.hpp"
+#include "gc/z/zJNICritical.hpp"
 #include "gc/z/zLargePages.hpp"
+#include "gc/z/zMarkStackAllocator.hpp"
 #include "gc/z/zNUMA.hpp"
 #include "gc/z/zStat.hpp"
 #include "gc/z/zThreadLocalAllocBuffer.hpp"
@@ -43,7 +47,7 @@ ZInitialize::ZInitialize(ZBarrierSet* barrier_set) {
                      VM_Version::jdk_debug_level());
 
   // Early initialization
-  ZAddress::initialize();
+  ZGlobalsPointers::initialize();
   ZNUMA::initialize();
   ZCPU::initialize();
   ZStatValue::initialize();
@@ -52,6 +56,9 @@ ZInitialize::ZInitialize(ZBarrierSet* barrier_set) {
   ZLargePages::initialize();
   ZHeuristics::set_medium_page_size();
   ZBarrierSet::set_barrier_set(barrier_set);
+  ZJNICritical::initialize();
+  ZDriver::initialize();
+  ZGCIdPrinter::initialize();
 
   pd_initialize();
 }
diff --git a/src/hotspot/share/gc/z/zIterator.hpp b/src/hotspot/share/gc/z/zIterator.hpp
new file mode 100644
index 00000000000..3a1de049dd0
--- /dev/null
+++ b/src/hotspot/share/gc/z/zIterator.hpp
@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZITERATOR_HPP
+#define SHARE_GC_Z_ZITERATOR_HPP
+
+#include "memory/allocation.hpp"
+#include "memory/iterator.hpp"
+
+class ZIterator : AllStatic {
+private:
+  static bool is_invisible_object(oop obj);
+  static bool is_invisible_object_array(oop obj);
+
+public:
+  // This iterator skips invisible roots
+  template <typename OopClosureT>
+  static void oop_iterate_safe(oop obj, OopClosureT* cl);
+
+  template <typename OopClosureT>
+  static void oop_iterate(oop obj, OopClosureT* cl);
+
+  template <typename OopClosureT>
+  static void oop_iterate_range(objArrayOop obj, OopClosureT* cl, int start, int end);
+
+  // This function skips invisible roots
+  template <typename Function>
+  static void basic_oop_iterate_safe(oop obj, Function function);
+
+  template <typename Function>
+  static void basic_oop_iterate(oop obj, Function function);
+};
+
+template <typename Function>
+class ZObjectClosure : public ObjectClosure {
+private:
+  Function _function;
+
+public:
+  ZObjectClosure(Function function);
+  virtual void do_object(oop obj);
+};
+
+#endif // SHARE_GC_Z_ZITERATOR_HPP
diff --git a/src/hotspot/share/gc/z/zIterator.inline.hpp b/src/hotspot/share/gc/z/zIterator.inline.hpp
new file mode 100644
index 00000000000..c6a388b9809
--- /dev/null
+++ b/src/hotspot/share/gc/z/zIterator.inline.hpp
@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZITERATOR_INLINE_HPP
+#define SHARE_GC_Z_ZITERATOR_INLINE_HPP
+
+#include "gc/z/zIterator.hpp"
+
+#include "memory/iterator.inline.hpp"
+#include "oops/objArrayOop.hpp"
+#include "oops/oop.inline.hpp"
+
+inline bool ZIterator::is_invisible_object(oop obj) {
+  return obj->mark_acquire().is_marked();
+}
+
+inline bool ZIterator::is_invisible_object_array(oop obj) {
+  return obj->klass()->is_objArray_klass() && is_invisible_object(obj);
+}
+
+// This iterator skips invisible object arrays
+template <typename OopClosureT>
+void ZIterator::oop_iterate_safe(oop obj, OopClosureT* cl) {
+  // Skip invisible object arrays - we only filter out *object* arrays,
+  // because that check is arguably faster than the is_invisible_object
+  // check, and primitive arrays are cheap to call oop_iterate on.
+  if (!is_invisible_object_array(obj)) {
+    obj->oop_iterate(cl);
+  }
+}
+
+template <typename OopClosureT>
+void ZIterator::oop_iterate(oop obj, OopClosureT* cl) {
+  assert(!is_invisible_object_array(obj), "not safe");
+  obj->oop_iterate(cl);
+}
+
+template <typename OopClosureT>
+void ZIterator::oop_iterate_range(objArrayOop obj, OopClosureT* cl, int start, int end) {
+  assert(!is_invisible_object_array(obj), "not safe");
+  obj->oop_iterate_range(cl, start, end);
+}
+
+template <typename Function>
+class ZBasicOopIterateClosure : public BasicOopIterateClosure {
+private:
+  Function _function;
+
+public:
+  ZBasicOopIterateClosure(Function function) :
+      _function(function) {}
+
+  virtual void do_oop(oop* p) {
+    _function((volatile zpointer*)p);
+  }
+
+  virtual void do_oop(narrowOop* p_) {
+    ShouldNotReachHere();
+  }
+};
+
+// This function skips invisible roots
+template <typename Function>
+void ZIterator::basic_oop_iterate_safe(oop obj, Function function) {
+  ZBasicOopIterateClosure<Function> cl(function);
+  oop_iterate_safe(obj, &cl);
+}
+
+template <typename Function>
+void ZIterator::basic_oop_iterate(oop obj, Function function) {
+  ZBasicOopIterateClosure<Function> cl(function);
+  oop_iterate(obj, &cl);
+}
+
+template <typename Function>
+ZObjectClosure<Function>::ZObjectClosure(Function function) :
+    _function(function) {}
+
+template <typename Function>
+void ZObjectClosure<Function>::do_object(oop obj) {
+  _function(obj);
+}
+
+#endif // SHARE_GC_Z_ZITERATOR_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zJNICritical.cpp b/src/hotspot/share/gc/z/zJNICritical.cpp
new file mode 100644
index 00000000000..d096367e12f
--- /dev/null
+++ b/src/hotspot/share/gc/z/zJNICritical.cpp
@@ -0,0 +1,183 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zJNICritical.hpp"
+#include "gc/z/zLock.inline.hpp"
+#include "gc/z/zStat.hpp"
+#include "runtime/atomic.hpp"
+#include "runtime/interfaceSupport.inline.hpp"
+#include "runtime/thread.inline.hpp"
+#include "utilities/debug.hpp"
+
+//
+// The JNI critical count reflects number of Java threads currently
+// inside a JNI critical region.
+//
+// * Normal (count >= 0). Java threads are allowed to enter and exit
+//   a critical region.
+//
+// * Blocked (count == -1). No Java thread is inside a critical region,
+//   and no Java thread can enter a critical region.
+//
+// * Block in progress (count < -1). Java threads are only allowed
+//   to exit a critical region. Attempts to enter a critical region
+//   will be blocked.
+//
+
+static const ZStatCriticalPhase ZCriticalPhaseJNICriticalStall("JNI Critical Stall", false /* verbose */);
+
+volatile int64_t ZJNICritical::_count;
+ZConditionLock*  ZJNICritical::_lock;
+
+void ZJNICritical::initialize() {
+  _count = 0;
+  _lock = new ZConditionLock();
+}
+
+void ZJNICritical::block() {
+  for (;;) {
+    const int64_t count = Atomic::load_acquire(&_count);
+
+    if (count < 0) {
+      // Already blocked, wait until unblocked
+      ZLocker<ZConditionLock> locker(_lock);
+      while (Atomic::load_acquire(&_count) < 0) {
+        _lock->wait();
+      }
+
+      // Unblocked
+      continue;
+    }
+
+    // Increment and invert count
+    if (Atomic::cmpxchg(&_count, count, -(count + 1)) != count) {
+      continue;
+    }
+
+    // If the previous count was 0, then we just incremented and inverted
+    // it to -1 and we have now blocked. Otherwise we wait until all Java
+    // threads have exited the critical region.
+    if (count != 0) {
+      // Wait until blocked
+      ZLocker<ZConditionLock> locker(_lock);
+      while (Atomic::load_acquire(&_count) != -1) {
+        _lock->wait();
+      }
+    }
+
+    // Blocked
+    return;
+  }
+}
+
+void ZJNICritical::unblock() {
+  const int64_t count = Atomic::load_acquire(&_count);
+  assert(count == -1, "Invalid count");
+
+  // Notify unblocked
+  ZLocker<ZConditionLock> locker(_lock);
+  Atomic::release_store(&_count, (int64_t)0);
+  _lock->notify_all();
+}
+
+void ZJNICritical::enter_inner(JavaThread* thread) {
+  for (;;) {
+    const int64_t count = Atomic::load_acquire(&_count);
+
+    if (count < 0) {
+      // Wait until unblocked
+      ZStatTimer timer(ZCriticalPhaseJNICriticalStall);
+
+      // Transition thread to blocked before locking to avoid deadlock
+      ThreadBlockInVM tbivm(thread);
+
+      ZLocker<ZConditionLock> locker(_lock);
+      while (Atomic::load_acquire(&_count) < 0) {
+        _lock->wait();
+      }
+
+      // Unblocked
+      continue;
+    }
+
+    // Increment count
+    if (Atomic::cmpxchg(&_count, count, count + 1) != count) {
+      continue;
+    }
+
+    // Entered critical region
+    return;
+  }
+}
+
+void ZJNICritical::enter(JavaThread* thread) {
+  assert(thread == JavaThread::current(), "Must be this thread");
+
+  if (!thread->in_critical()) {
+    enter_inner(thread);
+  }
+
+  thread->enter_critical();
+}
+
+void ZJNICritical::exit_inner() {
+  for (;;) {
+    const int64_t count = Atomic::load_acquire(&_count);
+    assert(count != 0, "Invalid count");
+
+    if (count > 0) {
+      // No block in progress, decrement count
+      if (Atomic::cmpxchg(&_count, count, count - 1) != count) {
+        continue;
+      }
+    } else {
+      // Block in progress, increment count
+      if (Atomic::cmpxchg(&_count, count, count + 1) != count) {
+        continue;
+      }
+
+      // If the previous count was -2, then we just incremented it to -1,
+      // and we should signal that all Java threads have now exited the
+      // critical region and we are now blocked.
+      if (count == -2) {
+        // Nofity blocked
+        ZLocker<ZConditionLock> locker(_lock);
+        _lock->notify_all();
+      }
+    }
+
+    // Exited critical region
+    return;
+  }
+}
+
+void ZJNICritical::exit(JavaThread* thread) {
+  assert(thread == JavaThread::current(), "Must be this thread");
+
+  thread->exit_critical();
+
+  if (!thread->in_critical()) {
+    exit_inner();
+  }
+}
diff --git a/src/hotspot/share/gc/z/zJNICritical.hpp b/src/hotspot/share/gc/z/zJNICritical.hpp
new file mode 100644
index 00000000000..d2ba80eddba
--- /dev/null
+++ b/src/hotspot/share/gc/z/zJNICritical.hpp
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZJNICRITICAL_HPP
+#define SHARE_GC_Z_ZJNICRITICAL_HPP
+
+#include "memory/allocation.hpp"
+
+class JavaThread;
+class ZConditionLock;
+
+class ZJNICritical : public AllStatic {
+private:
+  static volatile int64_t _count;
+  static ZConditionLock*  _lock;
+
+  static void enter_inner(JavaThread* thread);
+  static void exit_inner();
+
+public:
+  // For use by GC
+  static void initialize();
+  static void block();
+  static void unblock();
+
+  // For use by Java threads
+  static void enter(JavaThread* thread);
+  static void exit(JavaThread* thread);
+};
+
+#endif // SHARE_GC_Z_ZJNICRITICAL_HPP
diff --git a/src/hotspot/share/gc/z/zList.inline.hpp b/src/hotspot/share/gc/z/zList.inline.hpp
index f559f3a2d45..dcbcd87c6c6 100644
--- a/src/hotspot/share/gc/z/zList.inline.hpp
+++ b/src/hotspot/share/gc/z/zList.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -110,12 +110,12 @@ inline bool ZList<T>::is_empty() const {
 
 template <typename T>
 inline T* ZList<T>::first() const {
-  return is_empty() ? NULL : cast_to_outer(_head._next);
+  return is_empty() ? nullptr : cast_to_outer(_head._next);
 }
 
 template <typename T>
 inline T* ZList<T>::last() const {
-  return is_empty() ? NULL : cast_to_outer(_head._prev);
+  return is_empty() ? nullptr : cast_to_outer(_head._prev);
 }
 
 template <typename T>
@@ -128,7 +128,7 @@ inline T* ZList<T>::next(T* elem) const {
   ZListNode<T>* const next = node->_next;
   next->verify_links_linked();
 
-  return (next == &_head) ? NULL : cast_to_outer(next);
+  return (next == &_head) ? nullptr : cast_to_outer(next);
 }
 
 template <typename T>
@@ -141,7 +141,7 @@ inline T* ZList<T>::prev(T* elem) const {
   ZListNode<T>* const prev = node->_prev;
   prev->verify_links_linked();
 
-  return (prev == &_head) ? NULL : cast_to_outer(prev);
+  return (prev == &_head) ? nullptr : cast_to_outer(prev);
 }
 
 template <typename T>
@@ -191,7 +191,7 @@ inline void ZList<T>::remove(T* elem) {
 template <typename T>
 inline T* ZList<T>::remove_first() {
   T* elem = first();
-  if (elem != NULL) {
+  if (elem != nullptr) {
     remove(elem);
   }
 
@@ -201,7 +201,7 @@ inline T* ZList<T>::remove_first() {
 template <typename T>
 inline T* ZList<T>::remove_last() {
   T* elem = last();
-  if (elem != NULL) {
+  if (elem != nullptr) {
     remove(elem);
   }
 
@@ -215,7 +215,7 @@ inline ZListIteratorImpl<T, Forward>::ZListIteratorImpl(const ZList<T>* list) :
 
 template <typename T, bool Forward>
 inline bool ZListIteratorImpl<T, Forward>::next(T** elem) {
-  if (_next != NULL) {
+  if (_next != nullptr) {
     *elem = _next;
     _next = Forward ? _list->next(_next) : _list->prev(_next);
     return true;
@@ -232,7 +232,7 @@ inline ZListRemoveIteratorImpl<T, Forward>::ZListRemoveIteratorImpl(ZList<T>* li
 template <typename T, bool Forward>
 inline bool ZListRemoveIteratorImpl<T, Forward>::next(T** elem) {
   *elem = Forward ? _list->remove_first() : _list->remove_last();
-  return *elem != NULL;
+  return *elem != nullptr;
 }
 
 #endif // SHARE_GC_Z_ZLIST_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zLiveMap.cpp b/src/hotspot/share/gc/z/zLiveMap.cpp
index 7e04b7e8c37..0a3f61bad4c 100644
--- a/src/hotspot/share/gc/z/zLiveMap.cpp
+++ b/src/hotspot/share/gc/z/zLiveMap.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,10 +22,11 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zLiveMap.inline.hpp"
 #include "gc/z/zStat.hpp"
-#include "gc/z/zThread.inline.hpp"
+#include "gc/z/zUtils.hpp"
 #include "logging/log.hpp"
 #include "runtime/atomic.hpp"
 #include "utilities/debug.hpp"
@@ -48,14 +49,15 @@ ZLiveMap::ZLiveMap(uint32_t size) :
     _bitmap(bitmap_size(size, nsegments)),
     _segment_shift(exact_log2(segment_size())) {}
 
-void ZLiveMap::reset(size_t index) {
+void ZLiveMap::reset(ZGenerationId id) {
+  ZGeneration* const generation = ZGeneration::generation(id);
   const uint32_t seqnum_initializing = (uint32_t)-1;
   bool contention = false;
 
   // Multiple threads can enter here, make sure only one of them
   // resets the marking information while the others busy wait.
   for (uint32_t seqnum = Atomic::load_acquire(&_seqnum);
-       seqnum != ZGlobalSeqNum;
+       seqnum != generation->seqnum();
        seqnum = Atomic::load_acquire(&_seqnum)) {
     if ((seqnum != seqnum_initializing) &&
         (Atomic::cmpxchg(&_seqnum, seqnum, seqnum_initializing) == seqnum)) {
@@ -73,7 +75,7 @@ void ZLiveMap::reset(size_t index) {
       // before the update of the page seqnum, such that when the
       // up-to-date seqnum is load acquired, the bit maps will not
       // contain stale information.
-      Atomic::release_store(&_seqnum, ZGlobalSeqNum);
+      Atomic::release_store(&_seqnum, generation->seqnum());
       break;
     }
 
@@ -83,8 +85,8 @@ void ZLiveMap::reset(size_t index) {
       ZStatInc(ZCounterMarkSeqNumResetContention);
       contention = true;
 
-      log_trace(gc)("Mark seqnum reset contention, thread: " PTR_FORMAT " (%s), map: " PTR_FORMAT ", bit: " SIZE_FORMAT,
-                    ZThread::id(), ZThread::name(), p2i(this), index);
+      log_trace(gc)("Mark seqnum reset contention, thread: " PTR_FORMAT " (%s), map: " PTR_FORMAT,
+                    p2i(Thread::current()), ZUtils::thread_name(), p2i(this));
     }
   }
 }
@@ -102,7 +104,7 @@ void ZLiveMap::reset_segment(BitMap::idx_t segment) {
         contention = true;
 
         log_trace(gc)("Mark segment reset contention, thread: " PTR_FORMAT " (%s), map: " PTR_FORMAT ", segment: " SIZE_FORMAT,
-                      ZThread::id(), ZThread::name(), p2i(this), segment);
+                      p2i(Thread::current()), ZUtils::thread_name(), p2i(this), segment);
       }
     }
 
diff --git a/src/hotspot/share/gc/z/zLiveMap.hpp b/src/hotspot/share/gc/z/zLiveMap.hpp
index 07ae862876f..e3bcd2e267d 100644
--- a/src/hotspot/share/gc/z/zLiveMap.hpp
+++ b/src/hotspot/share/gc/z/zLiveMap.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,7 +24,9 @@
 #ifndef SHARE_GC_Z_ZLIVEMAP_HPP
 #define SHARE_GC_Z_ZLIVEMAP_HPP
 
+#include "gc/z/zAddress.hpp"
 #include "gc/z/zBitMap.hpp"
+#include "gc/z/zGenerationId.hpp"
 #include "memory/allocation.hpp"
 
 class ObjectClosure;
@@ -63,28 +65,36 @@ private:
 
   bool claim_segment(BitMap::idx_t segment);
 
-  void reset(size_t index);
+  void reset(ZGenerationId id);
   void reset_segment(BitMap::idx_t segment);
 
-  void iterate_segment(ObjectClosure* cl, BitMap::idx_t segment, uintptr_t page_start, size_t page_object_alignment_shift);
+  size_t do_object(ObjectClosure* cl, zaddress addr) const;
+
+  template <typename Function>
+  void iterate_segment(BitMap::idx_t segment, Function function);
 
 public:
   ZLiveMap(uint32_t size);
+  ZLiveMap(const ZLiveMap& other) = delete;
 
   void reset();
   void resize(uint32_t size);
 
-  bool is_marked() const;
+  bool is_marked(ZGenerationId id) const;
 
   uint32_t live_objects() const;
   size_t live_bytes() const;
 
-  bool get(size_t index) const;
-  bool set(size_t index, bool finalizable, bool& inc_live);
+  bool get(ZGenerationId id, BitMap::idx_t index) const;
+  bool set(ZGenerationId id, BitMap::idx_t index, bool finalizable, bool& inc_live);
 
   void inc_live(uint32_t objects, size_t bytes);
 
-  void iterate(ObjectClosure* cl, uintptr_t page_start, size_t page_object_alignment_shift);
+  template <typename Function>
+  void iterate(ZGenerationId id, Function function);
+
+  BitMap::idx_t find_base_bit(BitMap::idx_t index);
+  BitMap::idx_t find_base_bit_in_segment(BitMap::idx_t start, BitMap::idx_t index);
 };
 
 #endif // SHARE_GC_Z_ZLIVEMAP_HPP
diff --git a/src/hotspot/share/gc/z/zLiveMap.inline.hpp b/src/hotspot/share/gc/z/zLiveMap.inline.hpp
index b6d6f13367a..28390b72a89 100644
--- a/src/hotspot/share/gc/z/zLiveMap.inline.hpp
+++ b/src/hotspot/share/gc/z/zLiveMap.inline.hpp
@@ -26,9 +26,10 @@
 
 #include "gc/z/zLiveMap.hpp"
 
+#include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zBitMap.inline.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zMark.hpp"
-#include "gc/z/zOop.inline.hpp"
 #include "gc/z/zUtils.inline.hpp"
 #include "runtime/atomic.hpp"
 #include "utilities/bitMap.inline.hpp"
@@ -38,17 +39,15 @@ inline void ZLiveMap::reset() {
   _seqnum = 0;
 }
 
-inline bool ZLiveMap::is_marked() const {
-  return Atomic::load_acquire(&_seqnum) == ZGlobalSeqNum;
+inline bool ZLiveMap::is_marked(ZGenerationId id) const {
+  return Atomic::load_acquire(&_seqnum) == ZGeneration::generation(id)->seqnum();
 }
 
 inline uint32_t ZLiveMap::live_objects() const {
-  assert(ZGlobalPhase != ZPhaseMark, "Invalid phase");
   return _live_objects;
 }
 
 inline size_t ZLiveMap::live_bytes() const {
-  assert(ZGlobalPhase != ZPhaseMark, "Invalid phase");
   return _live_bytes;
 }
 
@@ -96,18 +95,18 @@ inline BitMap::idx_t ZLiveMap::index_to_segment(BitMap::idx_t index) const {
   return index >> _segment_shift;
 }
 
-inline bool ZLiveMap::get(size_t index) const {
-  BitMap::idx_t segment = index_to_segment(index);
-  return is_marked() &&                               // Page is marked
+inline bool ZLiveMap::get(ZGenerationId id, BitMap::idx_t index) const {
+  const BitMap::idx_t segment = index_to_segment(index);
+  return is_marked(id) &&                             // Page is marked
          is_segment_live(segment) &&                  // Segment is marked
          _bitmap.par_at(index, memory_order_relaxed); // Object is marked
 }
 
-inline bool ZLiveMap::set(size_t index, bool finalizable, bool& inc_live) {
-  if (!is_marked()) {
+inline bool ZLiveMap::set(ZGenerationId id, BitMap::idx_t index, bool finalizable, bool& inc_live) {
+  if (!is_marked(id)) {
     // First object to be marked during this
     // cycle, reset marking information.
-    reset(index);
+    reset(id);
   }
 
   const BitMap::idx_t segment = index_to_segment(index);
@@ -133,43 +132,98 @@ inline BitMap::idx_t ZLiveMap::segment_end(BitMap::idx_t segment) const {
   return segment_start(segment) + segment_size();
 }
 
-inline void ZLiveMap::iterate_segment(ObjectClosure* cl, BitMap::idx_t segment, uintptr_t page_start, size_t page_object_alignment_shift) {
+inline size_t ZLiveMap::do_object(ObjectClosure* cl, zaddress addr) const {
+  // Get the size of the object before calling the closure, which
+  // might overwrite the object in case we are relocating in-place.
+  const size_t size = ZUtils::object_size(addr);
+
+  // Apply closure
+  cl->do_object(to_oop(addr));
+
+  return size;
+}
+
+template <typename Function>
+inline void ZLiveMap::iterate_segment(BitMap::idx_t segment, Function function) {
   assert(is_segment_live(segment), "Must be");
 
   const BitMap::idx_t start_index = segment_start(segment);
   const BitMap::idx_t end_index   = segment_end(segment);
-  BitMap::idx_t index = _bitmap.find_first_set_bit(start_index, end_index);
 
-  while (index < end_index) {
-    // Calculate object address
-    const uintptr_t addr = page_start + ((index / 2) << page_object_alignment_shift);
+  _bitmap.iterate(function, start_index, end_index);
+}
 
-    // Get the size of the object before calling the closure, which
-    // might overwrite the object in case we are relocating in-place.
-    const size_t size = ZUtils::object_size(addr);
+template <typename Function>
+inline void ZLiveMap::iterate(ZGenerationId id, Function function) {
+  if (!is_marked(id)) {
+    return;
+  }
 
-    // Apply closure
-    cl->do_object(ZOop::from_address(addr));
-
-    // Find next bit after this object
-    const uintptr_t next_addr = align_up(addr + size, 1 << page_object_alignment_shift);
-    const BitMap::idx_t next_index = ((next_addr - page_start) >> page_object_alignment_shift) * 2;
-    if (next_index >= end_index) {
-      // End of live map
-      break;
+  auto live_only = [&](BitMap::idx_t index) -> bool {
+    if ((index & 1) == 0) {
+      return function(index);
     }
+    // Don't visit the finalizable bits
+    return true;
+  };
 
-    index = _bitmap.find_first_set_bit(next_index, end_index);
+  for (BitMap::idx_t segment = first_live_segment(); segment < nsegments; segment = next_live_segment(segment)) {
+    // For each live segment
+    iterate_segment(segment, live_only);
   }
 }
 
-inline void ZLiveMap::iterate(ObjectClosure* cl, uintptr_t page_start, size_t page_object_alignment_shift) {
-  if (is_marked()) {
-    for (BitMap::idx_t segment = first_live_segment(); segment < nsegments; segment = next_live_segment(segment)) {
-      // For each live segment
-      iterate_segment(cl, segment, page_start, page_object_alignment_shift);
+// Find the bit index that correspond the start of the object that is lower,
+// or equal, to the given index (index is inclusive).
+//
+// Typically used to find the start of an object when there's only a field
+// address available. Note that it's not guaranteed that the found index
+// corresponds to an object that spans the given index. This function just
+// looks at the bits. The calling code is responsible to check the object
+// at the returned index.
+//
+// returns -1 if no bit was found
+inline BitMap::idx_t ZLiveMap::find_base_bit(BitMap::idx_t index) {
+  // Check first segment
+  const BitMap::idx_t start_segment = index_to_segment(index);
+  if (is_segment_live(start_segment)) {
+    const BitMap::idx_t res = find_base_bit_in_segment(segment_start(start_segment), index);
+    if (res != BitMap::idx_t(-1)) {
+      return res;
     }
   }
+
+  // Search earlier segments
+  for (BitMap::idx_t segment = start_segment; segment-- > 0; ) {
+    if (is_segment_live(segment)) {
+      const BitMap::idx_t res = find_base_bit_in_segment(segment_start(segment), segment_end(segment) - 1);
+      if (res != BitMap::idx_t(-1)) {
+        return res;
+      }
+    }
+  }
+
+  // Not found
+  return BitMap::idx_t(-1);
+}
+
+// Find the bit index that correspond the start of the object that is lower,
+// or equal, to the given index (index is inclusive). Stopping when reaching
+// start.
+inline BitMap::idx_t ZLiveMap::find_base_bit_in_segment(BitMap::idx_t start, BitMap::idx_t index) {
+  assert(index_to_segment(start) == index_to_segment(index), "Only supports searches within segments start: %zu index: %zu", start, index);
+  assert(is_segment_live(index_to_segment(start)), "Must be live");
+
+  // Search backwards - + 1 to make an exclusive index.
+  const BitMap::idx_t end = index + 1;
+  const BitMap::idx_t bit = _bitmap.find_last_set_bit(start, end);
+  if (bit == end) {
+    return BitMap::idx_t(-1);
+  }
+
+  // The bitmaps contain pairs of bits to deal with strongly marked vs only
+  // finalizable marked. Align down to get the the first bit position.
+  return bit & ~BitMap::idx_t(1);
 }
 
 #endif // SHARE_GC_Z_ZLIVEMAP_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zLock.hpp b/src/hotspot/share/gc/z/zLock.hpp
index f069cb62391..640a9fb02d3 100644
--- a/src/hotspot/share/gc/z/zLock.hpp
+++ b/src/hotspot/share/gc/z/zLock.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,7 +27,7 @@
 #include "memory/allocation.hpp"
 #include "runtime/mutex.hpp"
 
-class ZLock {
+class ZLock : public CHeapObj<mtGC> {
 private:
   PlatformMutex _lock;
 
@@ -52,7 +52,7 @@ public:
   bool is_owned() const;
 };
 
-class ZConditionLock {
+class ZConditionLock : public CHeapObj<mtGC> {
 private:
   PlatformMonitor _lock;
 
diff --git a/src/hotspot/share/gc/z/zLock.inline.hpp b/src/hotspot/share/gc/z/zLock.inline.hpp
index d0df421b1ec..fb4b2d91cd7 100644
--- a/src/hotspot/share/gc/z/zLock.inline.hpp
+++ b/src/hotspot/share/gc/z/zLock.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -45,7 +45,7 @@ inline void ZLock::unlock() {
 
 inline ZReentrantLock::ZReentrantLock() :
     _lock(),
-    _owner(NULL),
+    _owner(nullptr),
     _count(0) {}
 
 inline void ZReentrantLock::lock() {
@@ -67,7 +67,7 @@ inline void ZReentrantLock::unlock() {
   _count--;
 
   if (_count == 0) {
-    Atomic::store(&_owner, (Thread*)NULL);
+    Atomic::store(&_owner, (Thread*)nullptr);
     _lock.unlock();
   }
 }
@@ -105,14 +105,14 @@ inline void ZConditionLock::notify_all() {
 template <typename T>
 inline ZLocker<T>::ZLocker(T* lock) :
     _lock(lock) {
-  if (_lock != NULL) {
+  if (_lock != nullptr) {
     _lock->lock();
   }
 }
 
 template <typename T>
 inline ZLocker<T>::~ZLocker() {
-  if (_lock != NULL) {
+  if (_lock != nullptr) {
     _lock->unlock();
   }
 }
diff --git a/src/hotspot/share/gc/z/zMark.cpp b/src/hotspot/share/gc/z/zMark.cpp
index e5b157f3a51..96f1576b83d 100644
--- a/src/hotspot/share/gc/z/zMark.cpp
+++ b/src/hotspot/share/gc/z/zMark.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,8 +30,13 @@
 #include "gc/shared/gc_globals.hpp"
 #include "gc/shared/stringdedup/stringDedup.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
+#include "gc/shared/workerThread.hpp"
 #include "gc/z/zAbort.inline.hpp"
+#include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zBarrier.inline.hpp"
+#include "gc/z/zBarrierSetNMethod.hpp"
+#include "gc/z/zGeneration.inline.hpp"
+#include "gc/z/zGenerationId.hpp"
 #include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zLock.inline.hpp"
 #include "gc/z/zMark.inline.hpp"
@@ -40,15 +45,14 @@
 #include "gc/z/zMarkStack.inline.hpp"
 #include "gc/z/zMarkTerminate.inline.hpp"
 #include "gc/z/zNMethod.hpp"
-#include "gc/z/zOop.inline.hpp"
 #include "gc/z/zPage.hpp"
 #include "gc/z/zPageTable.inline.hpp"
 #include "gc/z/zRootsIterator.hpp"
 #include "gc/z/zStackWatermark.hpp"
 #include "gc/z/zStat.hpp"
 #include "gc/z/zTask.hpp"
-#include "gc/z/zThread.inline.hpp"
 #include "gc/z/zThreadLocalAllocBuffer.hpp"
+#include "gc/z/zUncoloredRoot.inline.hpp"
 #include "gc/z/zUtils.inline.hpp"
 #include "gc/z/zWorkers.hpp"
 #include "logging/log.hpp"
@@ -64,23 +68,23 @@
 #include "runtime/stackWatermark.hpp"
 #include "runtime/stackWatermarkSet.inline.hpp"
 #include "runtime/threads.hpp"
+#include "runtime/vmThread.hpp"
 #include "utilities/align.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/powerOfTwo.hpp"
 #include "utilities/ticks.hpp"
 
-static const ZStatSubPhase ZSubPhaseConcurrentMark("Concurrent Mark");
-static const ZStatSubPhase ZSubPhaseConcurrentMarkTryFlush("Concurrent Mark Try Flush");
-static const ZStatSubPhase ZSubPhaseConcurrentMarkTryTerminate("Concurrent Mark Try Terminate");
-static const ZStatSubPhase ZSubPhaseMarkTryComplete("Pause Mark Try Complete");
+static const ZStatSubPhase ZSubPhaseConcurrentMarkRootUncoloredYoung("Concurrent Mark Root Uncolored", ZGenerationId::young);
+static const ZStatSubPhase ZSubPhaseConcurrentMarkRootColoredYoung("Concurrent Mark Root Colored", ZGenerationId::young);
+static const ZStatSubPhase ZSubPhaseConcurrentMarkRootUncoloredOld("Concurrent Mark Root Uncolored", ZGenerationId::old);
+static const ZStatSubPhase ZSubPhaseConcurrentMarkRootColoredOld("Concurrent Mark Root Colored", ZGenerationId::old);
 
-ZMark::ZMark(ZWorkers* workers, ZPageTable* page_table) :
-    _workers(workers),
+ZMark::ZMark(ZGeneration* generation, ZPageTable* page_table) :
+    _generation(generation),
     _page_table(page_table),
     _allocator(),
-    _stripes(),
+    _stripes(_allocator.start()),
     _terminate(),
-    _work_terminateflush(true),
     _work_nproactiveflush(0),
     _work_nterminateflush(0),
     _nproactiveflush(0),
@@ -107,16 +111,6 @@ void ZMark::start() {
     verify_all_stacks_empty();
   }
 
-  // Increment global sequence number to invalidate
-  // marking information for all pages.
-  ZGlobalSeqNum++;
-
-  // Note that we start a marking cycle.
-  // Unlike other GCs, the color switch implicitly changes the nmethods
-  // to be armed, and the thread-local disarm values are lazily updated
-  // when JavaThreads wake up from safepoints.
-  CodeCache::on_gc_marking_cycle_start();
-
   // Reset flush/continue counters
   _nproactiveflush = 0;
   _nterminateflush = 0;
@@ -124,7 +118,7 @@ void ZMark::start() {
   _ncontinue = 0;
 
   // Set number of workers to use
-  _nworkers = _workers->active_workers();
+  _nworkers = workers()->active_workers();
 
   // Set number of mark stripes to use, based on number
   // of workers we will use in the concurrent mark phase.
@@ -132,7 +126,7 @@ void ZMark::start() {
   _stripes.set_nstripes(nstripes);
 
   // Update statistics
-  ZStatMark::set_at_mark_start(nstripes);
+  _generation->stat_mark()->at_mark_start(nstripes);
 
   // Print worker/stripe distribution
   LogTarget(Debug, gc, marking) log;
@@ -147,15 +141,24 @@ void ZMark::start() {
   }
 }
 
+ZWorkers* ZMark::workers() const {
+  return _generation->workers();
+}
+
 void ZMark::prepare_work() {
-  assert(_nworkers == _workers->active_workers(), "Invalid number of workers");
+  // Set number of workers to use
+  _nworkers = workers()->active_workers();
+
+  // Set number of mark stripes to use, based on number
+  // of workers we will use in the concurrent mark phase.
+  const size_t nstripes = calculate_nstripes(_nworkers);
+  _stripes.set_nstripes(nstripes);
 
   // Set number of active workers
   _terminate.reset(_nworkers);
 
   // Reset flush counters
   _work_nproactiveflush = _work_nterminateflush = 0;
-  _work_terminateflush = true;
 }
 
 void ZMark::finish_work() {
@@ -164,143 +167,229 @@ void ZMark::finish_work() {
   _nterminateflush += _work_nterminateflush;
 }
 
-bool ZMark::is_array(uintptr_t addr) const {
-  return ZOop::from_address(addr)->is_objArray();
+void ZMark::follow_work_complete() {
+  follow_work(false /* partial */);
 }
 
-void ZMark::push_partial_array(uintptr_t addr, size_t size, bool finalizable) {
+bool ZMark::follow_work_partial() {
+  return follow_work(true /* partial */);
+}
+
+bool ZMark::is_array(zaddress addr) const {
+  return to_oop(addr)->is_objArray();
+}
+
+static uintptr_t encode_partial_array_offset(zpointer* addr) {
+  return untype(ZAddress::offset(to_zaddress((uintptr_t)addr))) >> ZMarkPartialArrayMinSizeShift;
+}
+
+static zpointer* decode_partial_array_offset(uintptr_t offset) {
+  return (zpointer*)ZOffset::address(to_zoffset(offset << ZMarkPartialArrayMinSizeShift));
+}
+
+void ZMark::push_partial_array(zpointer* addr, size_t length, bool finalizable) {
   assert(is_aligned(addr, ZMarkPartialArrayMinSize), "Address misaligned");
-  ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(Thread::current());
-  ZMarkStripe* const stripe = _stripes.stripe_for_addr(addr);
-  const uintptr_t offset = ZAddress::offset(addr) >> ZMarkPartialArrayMinSizeShift;
-  const uintptr_t length = size / oopSize;
+  ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::mark_stacks(Thread::current(), _generation->id());
+  ZMarkStripe* const stripe = _stripes.stripe_for_addr((uintptr_t)addr);
+  const uintptr_t offset = encode_partial_array_offset(addr);
   const ZMarkStackEntry entry(offset, length, finalizable);
 
   log_develop_trace(gc, marking)("Array push partial: " PTR_FORMAT " (" SIZE_FORMAT "), stripe: " SIZE_FORMAT,
-                                 addr, size, _stripes.stripe_id(stripe));
+                                 p2i(addr), length, _stripes.stripe_id(stripe));
 
-  stacks->push(&_allocator, &_stripes, stripe, entry, false /* publish */);
+  stacks->push(&_allocator, &_stripes, stripe, &_terminate, entry, false /* publish */);
 }
 
-void ZMark::follow_small_array(uintptr_t addr, size_t size, bool finalizable) {
-  assert(size <= ZMarkPartialArrayMinSize, "Too large, should be split");
-  const size_t length = size / oopSize;
-
-  log_develop_trace(gc, marking)("Array follow small: " PTR_FORMAT " (" SIZE_FORMAT ")", addr, size);
-
-  ZBarrier::mark_barrier_on_oop_array((oop*)addr, length, finalizable);
+static void mark_barrier_on_oop_array(volatile zpointer* p, size_t length, bool finalizable, bool young) {
+  for (volatile const zpointer* const end = p + length; p < end; p++) {
+    if (young) {
+      ZBarrier::mark_barrier_on_young_oop_field(p);
+    } else {
+      ZBarrier::mark_barrier_on_old_oop_field(p, finalizable);
+    }
+  }
 }
 
-void ZMark::follow_large_array(uintptr_t addr, size_t size, bool finalizable) {
-  assert(size <= (size_t)arrayOopDesc::max_array_length(T_OBJECT) * oopSize, "Too large");
-  assert(size > ZMarkPartialArrayMinSize, "Too small, should not be split");
-  const uintptr_t start = addr;
-  const uintptr_t end = start + size;
+void ZMark::follow_array_elements_small(zpointer* addr, size_t length, bool finalizable) {
+  assert(length <= ZMarkPartialArrayMinLength, "Too large, should be split");
+
+  log_develop_trace(gc, marking)("Array follow small: " PTR_FORMAT " (" SIZE_FORMAT ")", p2i(addr), length);
+
+  mark_barrier_on_oop_array(addr, length, finalizable, _generation->is_young());
+}
+
+void ZMark::follow_array_elements_large(zpointer* addr, size_t length, bool finalizable) {
+  assert(length <= (size_t)arrayOopDesc::max_array_length(T_OBJECT), "Too large");
+  assert(length > ZMarkPartialArrayMinLength, "Too small, should not be split");
+
+  zpointer* const start = addr;
+  zpointer* const end = start + length;
 
   // Calculate the aligned middle start/end/size, where the middle start
   // should always be greater than the start (hence the +1 below) to make
   // sure we always do some follow work, not just split the array into pieces.
-  const uintptr_t middle_start = align_up(start + 1, ZMarkPartialArrayMinSize);
-  const size_t    middle_size = align_down(end - middle_start, ZMarkPartialArrayMinSize);
-  const uintptr_t middle_end = middle_start + middle_size;
+  zpointer* const middle_start = align_up(start + 1, ZMarkPartialArrayMinSize);
+  const size_t    middle_length = align_down(end - middle_start, ZMarkPartialArrayMinLength);
+  zpointer* const middle_end = middle_start + middle_length;
 
   log_develop_trace(gc, marking)("Array follow large: " PTR_FORMAT "-" PTR_FORMAT" (" SIZE_FORMAT "), "
                                  "middle: " PTR_FORMAT "-" PTR_FORMAT " (" SIZE_FORMAT ")",
-                                 start, end, size, middle_start, middle_end, middle_size);
+                                 p2i(start), p2i(end), length, p2i(middle_start), p2i(middle_end), middle_length);
 
   // Push unaligned trailing part
   if (end > middle_end) {
-    const uintptr_t trailing_addr = middle_end;
-    const size_t trailing_size = end - middle_end;
-    push_partial_array(trailing_addr, trailing_size, finalizable);
+    zpointer* const trailing_addr = middle_end;
+    const size_t trailing_length = end - middle_end;
+    push_partial_array(trailing_addr, trailing_length, finalizable);
   }
 
   // Push aligned middle part(s)
-  uintptr_t partial_addr = middle_end;
+  zpointer* partial_addr = middle_end;
   while (partial_addr > middle_start) {
     const size_t parts = 2;
-    const size_t partial_size = align_up((partial_addr - middle_start) / parts, ZMarkPartialArrayMinSize);
-    partial_addr -= partial_size;
-    push_partial_array(partial_addr, partial_size, finalizable);
+    const size_t partial_length = align_up((partial_addr - middle_start) / parts, ZMarkPartialArrayMinLength);
+    partial_addr -= partial_length;
+    push_partial_array(partial_addr, partial_length, finalizable);
   }
 
   // Follow leading part
   assert(start < middle_start, "Miscalculated middle start");
-  const uintptr_t leading_addr = start;
-  const size_t leading_size = middle_start - start;
-  follow_small_array(leading_addr, leading_size, finalizable);
+  zpointer* const leading_addr = start;
+  const size_t leading_length = middle_start - start;
+  follow_array_elements_small(leading_addr, leading_length, finalizable);
 }
 
-void ZMark::follow_array(uintptr_t addr, size_t size, bool finalizable) {
-  if (size <= ZMarkPartialArrayMinSize) {
-    follow_small_array(addr, size, finalizable);
+void ZMark::follow_array_elements(zpointer* addr, size_t length, bool finalizable) {
+  if (length <= ZMarkPartialArrayMinLength) {
+    follow_array_elements_small(addr, length, finalizable);
   } else {
-    follow_large_array(addr, size, finalizable);
+    follow_array_elements_large(addr, length, finalizable);
   }
 }
 
 void ZMark::follow_partial_array(ZMarkStackEntry entry, bool finalizable) {
-  const uintptr_t addr = ZAddress::good(entry.partial_array_offset() << ZMarkPartialArrayMinSizeShift);
-  const size_t size = entry.partial_array_length() * oopSize;
+  zpointer* const addr = decode_partial_array_offset(entry.partial_array_offset());
+  const size_t length = entry.partial_array_length();
 
-  follow_array(addr, size, finalizable);
+  follow_array_elements(addr, length, finalizable);
 }
 
-template <bool finalizable>
-class ZMarkBarrierOopClosure : public ClaimMetadataVisitingOopIterateClosure {
+template <bool finalizable, ZGenerationIdOptional generation>
+class ZMarkBarrierFollowOopClosure : public OopIterateClosure {
+private:
+  static int claim_value() {
+    return finalizable ? ClassLoaderData::_claim_finalizable
+                       : ClassLoaderData::_claim_strong;
+  }
+
+  static ReferenceDiscoverer* discoverer() {
+    if (!finalizable) {
+      return ZGeneration::old()->reference_discoverer();
+    } else {
+      return nullptr;
+    }
+  }
+
+  static bool visit_metadata() {
+    // Only visit metadata if we're marking through the old generation
+    return ZGeneration::old()->is_phase_mark();
+  }
+
+  const bool _visit_metadata;
+
 public:
-  ZMarkBarrierOopClosure() :
-      ClaimMetadataVisitingOopIterateClosure(finalizable
-                                                 ? ClassLoaderData::_claim_finalizable
-                                                 : ClassLoaderData::_claim_strong,
-                                             finalizable
-                                                 ? NULL
-                                                 : ZHeap::heap()->reference_discoverer()) {}
+  ZMarkBarrierFollowOopClosure() :
+      OopIterateClosure(discoverer()),
+      _visit_metadata(visit_metadata()) {}
 
   virtual void do_oop(oop* p) {
-    ZBarrier::mark_barrier_on_oop_field(p, finalizable);
+    switch (generation) {
+    case ZGenerationIdOptional::young:
+      ZBarrier::mark_barrier_on_young_oop_field((volatile zpointer*)p);
+      break;
+    case ZGenerationIdOptional::old:
+      ZBarrier::mark_barrier_on_old_oop_field((volatile zpointer*)p, finalizable);
+      break;
+    case ZGenerationIdOptional::none:
+      ZBarrier::mark_barrier_on_oop_field((volatile zpointer*)p, finalizable);
+      break;
+    }
   }
 
   virtual void do_oop(narrowOop* p) {
     ShouldNotReachHere();
   }
 
+  virtual bool do_metadata() final {
+    // Only help out with metadata visiting
+    return _visit_metadata;
+  }
+
   virtual void do_nmethod(nmethod* nm) {
+    assert(do_metadata(), "Don't call otherwise");
     assert(!finalizable, "Can't handle finalizable marking of nmethods");
     nm->run_nmethod_entry_barrier();
   }
+
+  virtual void do_method(Method* m) {
+    // Mark interpreted frames for class redefinition
+    m->record_gc_epoch();
+  }
+
+  virtual void do_klass(Klass* klass) {
+    ClassLoaderData* cld = klass->class_loader_data();
+    ZMarkBarrierFollowOopClosure<finalizable, ZGenerationIdOptional::none> cl;
+    cld->oops_do(&cl, claim_value());
+  }
+
+  virtual void do_cld(ClassLoaderData* cld) {
+    ZMarkBarrierFollowOopClosure<finalizable, ZGenerationIdOptional::none> cl;
+    cld->oops_do(&cl, claim_value());
+  }
 };
 
 void ZMark::follow_array_object(objArrayOop obj, bool finalizable) {
-  if (finalizable) {
-    ZMarkBarrierOopClosure<true /* finalizable */> cl;
-    cl.do_klass(obj->klass());
+  if (_generation->is_old()) {
+    if (finalizable) {
+      ZMarkBarrierFollowOopClosure<true /* finalizable */, ZGenerationIdOptional::old> cl;
+      cl.do_klass(obj->klass());
+    } else {
+      ZMarkBarrierFollowOopClosure<false /* finalizable */, ZGenerationIdOptional::old> cl;
+      cl.do_klass(obj->klass());
+    }
   } else {
-    ZMarkBarrierOopClosure<false /* finalizable */> cl;
-    cl.do_klass(obj->klass());
+    ZMarkBarrierFollowOopClosure<false /* finalizable */, ZGenerationIdOptional::none> cl;
+    if (cl.do_metadata()) {
+      cl.do_klass(obj->klass());
+    }
   }
 
-  const uintptr_t addr = (uintptr_t)obj->base();
-  const size_t size = (size_t)obj->length() * oopSize;
+  // Should be convertible to colorless oop
+  assert_is_valid(to_zaddress(obj));
 
-  follow_array(addr, size, finalizable);
+  zpointer* const addr = (zpointer*)obj->base();
+  const size_t length = (size_t)obj->length();
+
+  follow_array_elements(addr, length, finalizable);
 }
 
 void ZMark::follow_object(oop obj, bool finalizable) {
-  if (ContinuationGCSupport::relativize_stack_chunk(obj)) {
-    // Loom doesn't support mixing of finalizable marking and strong marking of
-    // stack chunks. See: RelativizeDerivedOopClosure.
-    ZMarkBarrierOopClosure<false /* finalizable */> cl;
-    obj->oop_iterate(&cl);
-    return;
-  }
-
-  if (finalizable) {
-    ZMarkBarrierOopClosure<true /* finalizable */> cl;
-    obj->oop_iterate(&cl);
+  if (_generation->is_old()) {
+    if (ZHeap::heap()->is_old(to_zaddress(obj))) {
+      if (finalizable) {
+        ZMarkBarrierFollowOopClosure<true /* finalizable */, ZGenerationIdOptional::old> cl;
+        ZIterator::oop_iterate(obj, &cl);
+      } else {
+        ZMarkBarrierFollowOopClosure<false /* finalizable */, ZGenerationIdOptional::old> cl;
+        ZIterator::oop_iterate(obj, &cl);
+      }
+    } else {
+      fatal("Catch me!");
+    }
   } else {
-    ZMarkBarrierOopClosure<false /* finalizable */> cl;
-    obj->oop_iterate(&cl);
+    // Young gen must help out with old marking
+    ZMarkBarrierFollowOopClosure<false /* finalizable */, ZGenerationIdOptional::young> cl;
+    ZIterator::oop_iterate(obj, &cl);
   }
 }
 
@@ -335,7 +424,7 @@ void ZMark::mark_and_follow(ZMarkContext* context, ZMarkStackEntry entry) {
   }
 
   // Decode object address and additional flags
-  const uintptr_t addr = entry.object_address();
+  const zaddress addr = ZOffset::address(to_zoffset(entry.object_address()));
   const bool mark = entry.mark();
   bool inc_live = entry.inc_live();
   const bool follow = entry.follow();
@@ -362,9 +451,9 @@ void ZMark::mark_and_follow(ZMarkContext* context, ZMarkStackEntry entry) {
   // Follow
   if (follow) {
     if (is_array(addr)) {
-      follow_array_object(objArrayOop(ZOop::from_address(addr)), finalizable);
+      follow_array_object(objArrayOop(to_oop(addr)), finalizable);
     } else {
-      const oop obj = ZOop::from_address(addr);
+      const oop obj = to_oop(addr);
       follow_object(obj, finalizable);
 
       // Try deduplicate
@@ -373,25 +462,53 @@ void ZMark::mark_and_follow(ZMarkContext* context, ZMarkStackEntry entry) {
   }
 }
 
-template <typename T>
-bool ZMark::drain(ZMarkContext* context, T* timeout) {
-  ZMarkStripe* const stripe = context->stripe();
+// This function returns true if we need to stop working to resize threads or
+// abort marking
+bool ZMark::rebalance_work(ZMarkContext* context) {
+  const size_t assumed_nstripes = context->nstripes();
+  const size_t nstripes = _stripes.nstripes();
+
+  if (assumed_nstripes != nstripes) {
+    context->set_nstripes(nstripes);
+  } else if (nstripes < calculate_nstripes(_nworkers) && _allocator.clear_and_get_expanded_recently()) {
+    const size_t new_nstripes = nstripes << 1;
+    _stripes.set_nstripes(new_nstripes);
+    context->set_nstripes(new_nstripes);
+  }
+
+  ZMarkStripe* stripe = _stripes.stripe_for_worker(_nworkers, WorkerThread::worker_id());
+  if (context->stripe() != stripe) {
+    // Need to switch stripe
+    context->set_stripe(stripe);
+    flush_and_free();
+  } else if (!_terminate.saturated()) {
+    // Work imbalance detected; striped marking is likely going to be in the way
+    flush_and_free();
+  }
+
+  SuspendibleThreadSet::yield();
+
+  return ZAbort::should_abort() || _generation->should_worker_resize();
+}
+
+bool ZMark::drain(ZMarkContext* context) {
   ZMarkThreadLocalStacks* const stacks = context->stacks();
   ZMarkStackEntry entry;
+  size_t processed = 0;
+
+  context->set_stripe(_stripes.stripe_for_worker(_nworkers, WorkerThread::worker_id()));
+  context->set_nstripes(_stripes.nstripes());
 
   // Drain stripe stacks
-  while (stacks->pop(&_allocator, &_stripes, stripe, entry)) {
+  while (stacks->pop(&_allocator, &_stripes, context->stripe(), entry)) {
     mark_and_follow(context, entry);
 
-    // Check timeout
-    if (timeout->has_expired()) {
-      // Timeout
+    if ((processed++ & 31) == 0 && rebalance_work(context)) {
       return false;
     }
   }
 
-  // Success
-  return !timeout->has_expired();
+  return true;
 }
 
 bool ZMark::try_steal_local(ZMarkContext* context) {
@@ -403,7 +520,7 @@ bool ZMark::try_steal_local(ZMarkContext* context) {
        victim_stripe != stripe;
        victim_stripe = _stripes.stripe_next(victim_stripe)) {
     ZMarkStack* const stack = stacks->steal(&_stripes, victim_stripe);
-    if (stack != NULL) {
+    if (stack != nullptr) {
       // Success, install the stolen stack
       stacks->install(&_stripes, stripe, stack);
       return true;
@@ -423,7 +540,7 @@ bool ZMark::try_steal_global(ZMarkContext* context) {
        victim_stripe != stripe;
        victim_stripe = _stripes.stripe_next(victim_stripe)) {
     ZMarkStack* const stack = victim_stripe->steal_stack();
-    if (stack != NULL) {
+    if (stack != nullptr) {
       // Success, install the stolen stack
       stacks->install(&_stripes, stripe, stack);
       return true;
@@ -438,10 +555,6 @@ bool ZMark::try_steal(ZMarkContext* context) {
   return try_steal_local(context) || try_steal_global(context);
 }
 
-void ZMark::idle() const {
-  os::naked_short_sleep(1);
-}
-
 class ZMarkFlushAndFreeStacksClosure : public HandshakeClosure {
 private:
   ZMark* const _mark;
@@ -456,6 +569,9 @@ public:
   void do_thread(Thread* thread) {
     if (_mark->flush_and_free(thread)) {
       _flushed = true;
+      if (SafepointSynchronize::is_at_safepoint()) {
+        log_debug(gc, marking)("Thread broke mark termination %s", thread->name());
+      }
     }
   }
 
@@ -464,201 +580,125 @@ public:
   }
 };
 
-bool ZMark::flush(bool at_safepoint) {
-  ZMarkFlushAndFreeStacksClosure cl(this);
-  if (at_safepoint) {
-    Threads::threads_do(&cl);
-  } else {
-    Handshake::execute(&cl);
+class VM_ZMarkFlushOperation : public VM_Operation {
+private:
+  ThreadClosure* _cl;
+
+public:
+  VM_ZMarkFlushOperation(ThreadClosure* cl) :
+      _cl(cl) {}
+
+  virtual bool evaluate_at_safepoint() const {
+    return false;
   }
 
+  virtual void doit() {
+    // Flush VM thread
+    Thread* const thread = Thread::current();
+    _cl->do_thread(thread);
+  }
+
+  virtual VMOp_Type type() const {
+    return VMOp_ZMarkFlushOperation;
+  }
+};
+
+bool ZMark::flush() {
+  ZMarkFlushAndFreeStacksClosure cl(this);
+  VM_ZMarkFlushOperation vm_cl(&cl);
+  Handshake::execute(&cl);
+  VMThread::execute(&vm_cl);
+
   // Returns true if more work is available
   return cl.flushed() || !_stripes.is_empty();
 }
 
-bool ZMark::try_flush(volatile size_t* nflush) {
-  Atomic::inc(nflush);
+bool ZMark::try_terminate_flush() {
+  Atomic::inc(&_work_nterminateflush);
+  _terminate.set_resurrected(false);
 
-  ZStatTimer timer(ZSubPhaseConcurrentMarkTryFlush);
-  return flush(false /* at_safepoint */);
+  if (ZVerifyMarking) {
+    verify_worker_stacks_empty();
+  }
+
+  return flush() ||
+         _terminate.resurrected();
 }
 
 bool ZMark::try_proactive_flush() {
   // Only do proactive flushes from worker 0
-  if (ZThread::worker_id() != 0) {
+  if (WorkerThread::worker_id() != 0) {
     return false;
   }
 
-  if (Atomic::load(&_work_nproactiveflush) == ZMarkProactiveFlushMax ||
-      Atomic::load(&_work_nterminateflush) != 0) {
+  if (Atomic::load(&_work_nproactiveflush) == ZMarkProactiveFlushMax) {
     // Limit reached or we're trying to terminate
     return false;
   }
 
-  return try_flush(&_work_nproactiveflush);
+  Atomic::inc(&_work_nproactiveflush);
+
+  SuspendibleThreadSetLeaver sts_leaver;
+  return flush();
 }
 
-bool ZMark::try_terminate() {
-  ZStatTimer timer(ZSubPhaseConcurrentMarkTryTerminate);
-
-  if (_terminate.enter_stage0()) {
-    // Last thread entered stage 0, flush
-    if (Atomic::load(&_work_terminateflush) &&
-        Atomic::load(&_work_nterminateflush) != ZMarkTerminateFlushMax) {
-      // Exit stage 0 to allow other threads to continue marking
-      _terminate.exit_stage0();
-
-      // Flush before termination
-      if (!try_flush(&_work_nterminateflush)) {
-        // No more work available, skip further flush attempts
-        Atomic::store(&_work_terminateflush, false);
-      }
-
-      // Don't terminate, regardless of whether we successfully
-      // flushed out more work or not. We've already exited
-      // termination stage 0, to allow other threads to continue
-      // marking, so this thread has to return false and also
-      // make another round of attempted marking.
-      return false;
-    }
-  }
-
-  for (;;) {
-    if (_terminate.enter_stage1()) {
-      // Last thread entered stage 1, terminate
-      return true;
-    }
-
-    // Idle to give the other threads
-    // a chance to enter termination.
-    idle();
-
-    if (!_terminate.try_exit_stage1()) {
-      // All workers in stage 1, terminate
-      return true;
-    }
-
-    if (_terminate.try_exit_stage0()) {
-      // More work available, don't terminate
-      return false;
-    }
-  }
+bool ZMark::try_terminate(ZMarkContext* context) {
+  return _terminate.try_terminate(&_stripes, context->nstripes());
 }
 
-class ZMarkNoTimeout : public StackObj {
-public:
-  bool has_expired() {
-    // No timeout, but check for signal to abort
-    return ZAbort::should_abort();
-  }
-};
+void ZMark::leave() {
+  _terminate.leave();
+}
 
-void ZMark::work_without_timeout(ZMarkContext* context) {
-  ZStatTimer timer(ZSubPhaseConcurrentMark);
-  ZMarkNoTimeout no_timeout;
+// Returning true means marking finished successfully after marking as far as it could.
+// Returning false means that marking finished unsuccessfully due to abort or resizing.
+bool ZMark::follow_work(bool partial) {
+  ZMarkStripe* const stripe = _stripes.stripe_for_worker(_nworkers, WorkerThread::worker_id());
+  ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::mark_stacks(Thread::current(), _generation->id());
+  ZMarkContext context(ZMarkStripesMax, stripe, stacks);
 
   for (;;) {
-    if (!drain(context, &no_timeout)) {
-      // Abort
-      break;
+    if (!drain(&context)) {
+      leave();
+      return false;
     }
 
-    if (try_steal(context)) {
+    if (try_steal(&context)) {
       // Stole work
       continue;
     }
 
+    if (partial) {
+      return true;
+    }
+
     if (try_proactive_flush()) {
       // Work available
       continue;
     }
 
-    if (try_terminate()) {
+    if (try_terminate(&context)) {
       // Terminate
-      break;
+      return true;
     }
   }
 }
 
-class ZMarkTimeout : public StackObj {
-private:
-  const Ticks    _start;
-  const uint64_t _timeout;
-  const uint64_t _check_interval;
-  uint64_t       _check_at;
-  uint64_t       _check_count;
-  bool           _expired;
-
-public:
-  ZMarkTimeout(uint64_t timeout_in_micros) :
-      _start(Ticks::now()),
-      _timeout(_start.value() + TimeHelper::micros_to_counter(timeout_in_micros)),
-      _check_interval(200),
-      _check_at(_check_interval),
-      _check_count(0),
-      _expired(false) {}
-
-  ~ZMarkTimeout() {
-    const Tickspan duration = Ticks::now() - _start;
-    log_debug(gc, marking)("Mark With Timeout (%s): %s, " UINT64_FORMAT " oops, %.3fms",
-                           ZThread::name(), _expired ? "Expired" : "Completed",
-                           _check_count, TimeHelper::counter_to_millis(duration.value()));
-  }
-
-  bool has_expired() {
-    if (++_check_count == _check_at) {
-      _check_at += _check_interval;
-      if ((uint64_t)Ticks::now().value() >= _timeout) {
-        // Timeout
-        _expired = true;
-      }
-    }
-
-    return _expired;
-  }
-};
-
-void ZMark::work_with_timeout(ZMarkContext* context, uint64_t timeout_in_micros) {
-  ZStatTimer timer(ZSubPhaseMarkTryComplete);
-  ZMarkTimeout timeout(timeout_in_micros);
-
-  for (;;) {
-    if (!drain(context, &timeout)) {
-      // Timed out
-      break;
-    }
-
-    if (try_steal(context)) {
-      // Stole work
-      continue;
-    }
-
-    // Terminate
-    break;
-  }
-}
-
-void ZMark::work(uint64_t timeout_in_micros) {
-  ZMarkStripe* const stripe = _stripes.stripe_for_worker(_nworkers, ZThread::worker_id());
-  ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(Thread::current());
-  ZMarkContext context(_stripes.nstripes(), stripe, stacks);
-
-  if (timeout_in_micros == 0) {
-    work_without_timeout(&context);
-  } else {
-    work_with_timeout(&context, timeout_in_micros);
-  }
-
-  // Flush and publish stacks
-  stacks->flush(&_allocator, &_stripes);
-
-  // Free remaining stacks
-  stacks->free(&_allocator);
-}
-
 class ZMarkOopClosure : public OopClosure {
+public:
   virtual void do_oop(oop* p) {
-    ZBarrier::mark_barrier_on_oop_field(p, false /* finalizable */);
+    ZBarrier::mark_barrier_on_oop_field((zpointer*)p, false /* finalizable */);
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+};
+
+class ZMarkYoungOopClosure : public OopClosure {
+public:
+  virtual void do_oop(oop* p) {
+    ZBarrier::mark_young_good_barrier_on_oop_field((zpointer*)p);
   }
 
   virtual void do_oop(narrowOop* p) {
@@ -668,98 +708,222 @@ class ZMarkOopClosure : public OopClosure {
 
 class ZMarkThreadClosure : public ThreadClosure {
 private:
-  OopClosure* const _cl;
+  static ZUncoloredRoot::RootFunction root_function() {
+    return ZUncoloredRoot::mark;
+  }
 
 public:
-  ZMarkThreadClosure(OopClosure* cl) :
-      _cl(cl) {
+  ZMarkThreadClosure() {
     ZThreadLocalAllocBuffer::reset_statistics();
   }
   ~ZMarkThreadClosure() {
     ZThreadLocalAllocBuffer::publish_statistics();
   }
+
   virtual void do_thread(Thread* thread) {
     JavaThread* const jt = JavaThread::cast(thread);
-    StackWatermarkSet::finish_processing(jt, _cl, StackWatermarkKind::gc);
+
+    StackWatermarkSet::finish_processing(jt, (void*)root_function(), StackWatermarkKind::gc);
     ZThreadLocalAllocBuffer::update_stats(jt);
   }
 };
 
 class ZMarkNMethodClosure : public NMethodClosure {
 private:
-  OopClosure* const _cl;
+  ZBarrierSetNMethod* const _bs_nm;
 
 public:
-  ZMarkNMethodClosure(OopClosure* cl) :
-      _cl(cl) {}
+  ZMarkNMethodClosure() :
+      _bs_nm(static_cast<ZBarrierSetNMethod*>(BarrierSet::barrier_set()->barrier_set_nmethod())) {}
 
   virtual void do_nmethod(nmethod* nm) {
     ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
-    if (ZNMethod::is_armed(nm)) {
-      ZNMethod::nmethod_oops_do_inner(nm, _cl);
+    if (_bs_nm->is_armed(nm)) {
+      // Heal barriers
+      ZNMethod::nmethod_patch_barriers(nm);
+
+      // Heal oops
+      ZUncoloredRootMarkOopClosure cl(ZNMethod::color(nm));
+      ZNMethod::nmethod_oops_do_inner(nm, &cl);
 
       // CodeCache unloading support
       nm->mark_as_maybe_on_stack();
 
-      ZNMethod::disarm(nm);
+      log_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by old", p2i(nm));
+
+      // Disarm
+      _bs_nm->disarm(nm);
     }
   }
 };
 
-typedef ClaimingCLDToOopClosure<ClassLoaderData::_claim_strong> ZMarkCLDClosure;
-
-class ZMarkRootsTask : public ZTask {
+class ZMarkYoungNMethodClosure : public NMethodClosure {
 private:
-  ZMark* const               _mark;
-  SuspendibleThreadSetJoiner _sts_joiner;
-  ZRootsIterator             _roots;
-
-  ZMarkOopClosure            _cl;
-  ZMarkCLDClosure            _cld_cl;
-  ZMarkThreadClosure         _thread_cl;
-  ZMarkNMethodClosure        _nm_cl;
+  ZBarrierSetNMethod* const _bs_nm;
 
 public:
-  ZMarkRootsTask(ZMark* mark) :
-      ZTask("ZMarkRootsTask"),
+  ZMarkYoungNMethodClosure() :
+      _bs_nm(static_cast<ZBarrierSetNMethod*>(BarrierSet::barrier_set()->barrier_set_nmethod())) {}
+
+  virtual void do_nmethod(nmethod* nm) {
+    ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
+    if (nm->is_unloading()) {
+      return;
+    }
+
+    if (_bs_nm->is_armed(nm)) {
+      const uintptr_t prev_color = ZNMethod::color(nm);
+
+      // Heal oops
+      ZUncoloredRootMarkYoungOopClosure cl(prev_color);
+      ZNMethod::nmethod_oops_do_inner(nm, &cl);
+
+      // Disarm only the young marking, not any potential old marking cycle
+
+      const uintptr_t old_marked_mask = ZPointerMarkedMask ^ (ZPointerMarkedYoung0 | ZPointerMarkedYoung1);
+      const uintptr_t old_marked = prev_color & old_marked_mask;
+
+      const zpointer new_disarm_value_ptr = ZAddress::color(zaddress::null, ZPointerLoadGoodMask | ZPointerMarkedYoung | old_marked | ZPointerRemembered);
+
+      // Check if disarming for young mark, completely disarms the nmethod entry barrier
+      const bool complete_disarm = ZPointer::is_store_good(new_disarm_value_ptr);
+
+      if (complete_disarm) {
+        // We are about to completely disarm the nmethod, must take responsibility to patch all barriers before disarming
+        ZNMethod::nmethod_patch_barriers(nm);
+      }
+
+      _bs_nm->set_guard_value(nm, (int)untype(new_disarm_value_ptr));
+
+      if (complete_disarm) {
+        log_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by young (complete) [" PTR_FORMAT " -> " PTR_FORMAT "]", p2i(nm), prev_color, untype(new_disarm_value_ptr));
+        assert(!_bs_nm->is_armed(nm), "Must not be considered armed anymore");
+      } else {
+        log_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by young (incomplete) [" PTR_FORMAT " -> " PTR_FORMAT "]", p2i(nm), prev_color, untype(new_disarm_value_ptr));
+        assert(_bs_nm->is_armed(nm), "Must be considered armed");
+      }
+    }
+  }
+};
+
+typedef ClaimingCLDToOopClosure<ClassLoaderData::_claim_strong> ZMarkOldCLDClosure;
+
+class ZMarkOldRootsTask : public ZTask {
+private:
+  ZMark* const                  _mark;
+  ZRootsIteratorStrongColored   _roots_colored;
+  ZRootsIteratorStrongUncolored _roots_uncolored;
+
+  ZMarkOopClosure               _cl_colored;
+  ZMarkOldCLDClosure            _cld_cl;
+
+  ZMarkThreadClosure            _thread_cl;
+  ZMarkNMethodClosure           _nm_cl;
+
+public:
+  ZMarkOldRootsTask(ZMark* mark) :
+      ZTask("ZMarkOldRootsTask"),
       _mark(mark),
-      _sts_joiner(),
-      _roots(ClassLoaderData::_claim_strong),
-      _cl(),
-      _cld_cl(&_cl),
-      _thread_cl(&_cl),
-      _nm_cl(&_cl) {
+      _roots_colored(ZGenerationIdOptional::old),
+      _roots_uncolored(ZGenerationIdOptional::old),
+      _cl_colored(),
+      _cld_cl(&_cl_colored),
+      _thread_cl(),
+      _nm_cl() {
     ClassLoaderDataGraph_lock->lock();
   }
 
-  ~ZMarkRootsTask() {
+  ~ZMarkOldRootsTask() {
     ClassLoaderDataGraph_lock->unlock();
   }
 
   virtual void work() {
-    _roots.apply(&_cl,
-                 &_cld_cl,
-                 &_thread_cl,
-                 &_nm_cl);
+    {
+      ZStatTimerWorker timer(ZSubPhaseConcurrentMarkRootColoredOld);
+      _roots_colored.apply(&_cl_colored,
+                           &_cld_cl);
+    }
+
+    {
+      ZStatTimerWorker timer(ZSubPhaseConcurrentMarkRootUncoloredOld);
+      _roots_uncolored.apply(&_thread_cl,
+                             &_nm_cl);
+    }
 
     // Flush and free worker stacks. Needed here since
     // the set of workers executing during root scanning
     // can be different from the set of workers executing
     // during mark.
-    _mark->flush_and_free();
+    ZHeap::heap()->mark_flush_and_free(Thread::current());
   }
 };
 
-class ZMarkTask : public ZTask {
+class ZMarkYoungCLDClosure : public ClaimingCLDToOopClosure<ClassLoaderData::_claim_none> {
+public:
+  virtual void do_cld(ClassLoaderData* cld) {
+    if (!cld->is_alive()) {
+      // Skip marking through concurrently unloading CLDs
+      return;
+    }
+    ClaimingCLDToOopClosure<ClassLoaderData::_claim_none>::do_cld(cld);
+  }
+
+  ZMarkYoungCLDClosure(OopClosure* cl) :
+      ClaimingCLDToOopClosure<ClassLoaderData::_claim_none>(cl) {}
+};
+
+class ZMarkYoungRootsTask : public ZTask {
 private:
-  ZMark* const   _mark;
-  const uint64_t _timeout_in_micros;
+  ZMark* const               _mark;
+  ZRootsIteratorAllColored   _roots_colored;
+  ZRootsIteratorAllUncolored _roots_uncolored;
+
+  ZMarkYoungOopClosure       _cl_colored;
+  ZMarkYoungCLDClosure       _cld_cl;
+
+  ZMarkThreadClosure         _thread_cl;
+  ZMarkYoungNMethodClosure   _nm_cl;
 
 public:
-  ZMarkTask(ZMark* mark, uint64_t timeout_in_micros = 0) :
-      ZTask("ZMarkTask"),
+  ZMarkYoungRootsTask(ZMark* mark) :
+      ZTask("ZMarkYoungRootsTask"),
       _mark(mark),
-      _timeout_in_micros(timeout_in_micros) {
+      _roots_colored(ZGenerationIdOptional::young),
+      _roots_uncolored(ZGenerationIdOptional::young),
+      _cl_colored(),
+      _cld_cl(&_cl_colored),
+      _thread_cl(),
+      _nm_cl() {}
+
+  virtual void work() {
+    {
+      ZStatTimerWorker timer(ZSubPhaseConcurrentMarkRootColoredYoung);
+      _roots_colored.apply(&_cl_colored,
+                           &_cld_cl);
+    }
+
+    {
+      ZStatTimerWorker timer(ZSubPhaseConcurrentMarkRootUncoloredYoung);
+      _roots_uncolored.apply(&_thread_cl,
+                             &_nm_cl);
+    }
+
+    // Flush and free worker stacks. Needed here since
+    // the set of workers executing during root scanning
+    // can be different from the set of workers executing
+    // during mark.
+    ZHeap::heap()->mark_flush_and_free(Thread::current());
+  }
+};
+
+class ZMarkTask : public ZRestartableTask {
+private:
+  ZMark* const _mark;
+
+public:
+  ZMarkTask(ZMark* mark) :
+      ZRestartableTask("ZMarkTask"),
+      _mark(mark) {
     _mark->prepare_work();
   }
 
@@ -768,42 +932,66 @@ public:
   }
 
   virtual void work() {
-    _mark->work(_timeout_in_micros);
+    SuspendibleThreadSetJoiner sts_joiner;
+    _mark->follow_work_complete();
+    // We might have found pointers into the other generation, and then we want to
+    // publish such marking stacks to prevent that generation from getting a mark continue.
+    // We also flush in case of a resize where a new worker thread continues the marking
+    // work, causing a mark continue for the collected generation.
+    ZHeap::heap()->mark_flush_and_free(Thread::current());
+  }
+
+  virtual void resize_workers(uint nworkers) {
+    _mark->resize_workers(nworkers);
   }
 };
 
-void ZMark::mark(bool initial) {
-  if (initial) {
-    ZMarkRootsTask task(this);
-    _workers->run(&task);
-  }
-
-  ZMarkTask task(this);
-  _workers->run(&task);
+void ZMark::resize_workers(uint nworkers) {
+  _nworkers = nworkers;
+  const size_t nstripes = calculate_nstripes(nworkers);
+  _stripes.set_nstripes(nstripes);
+  _terminate.reset(nworkers);
 }
 
-bool ZMark::try_complete() {
-  _ntrycomplete++;
+void ZMark::mark_young_roots() {
+  SuspendibleThreadSetJoiner sts_joiner;
+  ZMarkYoungRootsTask task(this);
+  workers()->run(&task);
+}
 
-  // Use nconcurrent number of worker threads to maintain the
-  // worker/stripe distribution used during concurrent mark.
-  ZMarkTask task(this, ZMarkCompleteTimeout);
-  _workers->run(&task);
+void ZMark::mark_old_roots() {
+  SuspendibleThreadSetJoiner sts_joiner;
+  ZMarkOldRootsTask task(this);
+  workers()->run(&task);
+}
 
-  // Successful if all stripes are empty
-  return _stripes.is_empty();
+void ZMark::mark_follow() {
+  for (;;) {
+    ZMarkTask task(this);
+    workers()->run(&task);
+    if (ZAbort::should_abort() || !try_terminate_flush()) {
+      break;
+    }
+  }
 }
 
 bool ZMark::try_end() {
-  // Flush all mark stacks
-  if (!flush(true /* at_safepoint */)) {
-    // Mark completed
-    return true;
+  if (_terminate.resurrected()) {
+    // An oop was resurrected after concurrent termination.
+    return false;
   }
 
-  // Try complete marking by doing a limited
-  // amount of mark work in this phase.
-  return try_complete();
+  // Try end marking
+  ZMarkFlushAndFreeStacksClosure cl(this);
+  Threads::non_java_threads_do(&cl);
+
+  // Check if non-java threads have any pending marking
+  if (cl.flushed() || !_stripes.is_empty()) {
+    return false;
+  }
+
+  // Mark completed
+  return true;
 }
 
 bool ZMark::end() {
@@ -820,12 +1008,7 @@ bool ZMark::end() {
   }
 
   // Update statistics
-  ZStatMark::set_at_mark_end(_nproactiveflush, _nterminateflush, _ntrycomplete, _ncontinue);
-
-  // Note that we finished a marking cycle.
-  // Unlike other GCs, we do not arm the nmethods
-  // when marking terminates.
-  CodeCache::on_gc_marking_cycle_finish();
+  _generation->stat_mark()->at_mark_end(_nproactiveflush, _nterminateflush, _ntrycomplete, _ncontinue);
 
   // Mark completed
   return true;
@@ -836,7 +1019,7 @@ void ZMark::free() {
   _allocator.free();
 
   // Update statistics
-  ZStatMark::set_at_mark_free(_allocator.size());
+  _generation->stat_mark()->at_mark_free(_allocator.size());
 }
 
 void ZMark::flush_and_free() {
@@ -845,8 +1028,11 @@ void ZMark::flush_and_free() {
 }
 
 bool ZMark::flush_and_free(Thread* thread) {
-  ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(thread);
-  const bool flushed = stacks->flush(&_allocator, &_stripes);
+  if (thread->is_Java_thread()) {
+    ZThreadLocalData::store_barrier_buffer(thread)->flush();
+  }
+  ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::mark_stacks(thread, _generation->id());
+  const bool flushed = stacks->flush(&_allocator, &_stripes, &_terminate);
   stacks->free(&_allocator);
   return flushed;
 }
@@ -854,22 +1040,30 @@ bool ZMark::flush_and_free(Thread* thread) {
 class ZVerifyMarkStacksEmptyClosure : public ThreadClosure {
 private:
   const ZMarkStripeSet* const _stripes;
+  const ZGenerationId _generation_id;
 
 public:
-  ZVerifyMarkStacksEmptyClosure(const ZMarkStripeSet* stripes) :
-      _stripes(stripes) {}
+  ZVerifyMarkStacksEmptyClosure(const ZMarkStripeSet* stripes, ZGenerationId id) :
+      _stripes(stripes),
+      _generation_id(id) {}
 
   void do_thread(Thread* thread) {
-    ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(thread);
+    ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::mark_stacks(thread, _generation_id);
     guarantee(stacks->is_empty(_stripes), "Should be empty");
   }
 };
 
 void ZMark::verify_all_stacks_empty() const {
   // Verify thread stacks
-  ZVerifyMarkStacksEmptyClosure cl(&_stripes);
+  ZVerifyMarkStacksEmptyClosure cl(&_stripes, _generation->id());
   Threads::threads_do(&cl);
 
   // Verify stripe stacks
   guarantee(_stripes.is_empty(), "Should be empty");
 }
+
+void ZMark::verify_worker_stacks_empty() const {
+  // Verify thread stacks
+  ZVerifyMarkStacksEmptyClosure cl(&_stripes, _generation->id());
+  workers()->threads_do(&cl);
+}
diff --git a/src/hotspot/share/gc/z/zMark.hpp b/src/hotspot/share/gc/z/zMark.hpp
index 1de4eb604ea..552bf3b959d 100644
--- a/src/hotspot/share/gc/z/zMark.hpp
+++ b/src/hotspot/share/gc/z/zMark.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
 #ifndef SHARE_GC_Z_ZMARK_HPP
 #define SHARE_GC_Z_ZMARK_HPP
 
+#include "gc/z/zAddress.hpp"
 #include "gc/z/zMarkStack.hpp"
 #include "gc/z/zMarkStackAllocator.hpp"
 #include "gc/z/zMarkStackEntry.hpp"
@@ -32,6 +33,7 @@
 #include "utilities/globalDefinitions.hpp"
 
 class Thread;
+class ZGeneration;
 class ZMarkContext;
 class ZPageTable;
 class ZWorkers;
@@ -39,13 +41,25 @@ class ZWorkers;
 class ZMark {
   friend class ZMarkTask;
 
+public:
+  static const bool Resurrect     = true;
+  static const bool DontResurrect = false;
+
+  static const bool GCThread      = true;
+  static const bool AnyThread     = false;
+
+  static const bool Follow        = true;
+  static const bool DontFollow    = false;
+
+  static const bool Strong        = false;
+  static const bool Finalizable   = true;
+
 private:
-  ZWorkers* const     _workers;
+  ZGeneration* const  _generation;
   ZPageTable* const   _page_table;
   ZMarkStackAllocator _allocator;
   ZMarkStripeSet      _stripes;
   ZMarkTerminate      _terminate;
-  volatile bool       _work_terminateflush;
   volatile size_t     _work_nproactiveflush;
   volatile size_t     _work_nterminateflush;
   size_t              _nproactiveflush;
@@ -56,51 +70,59 @@ private:
 
   size_t calculate_nstripes(uint nworkers) const;
 
-  bool is_array(uintptr_t addr) const;
-  void push_partial_array(uintptr_t addr, size_t size, bool finalizable);
-  void follow_small_array(uintptr_t addr, size_t size, bool finalizable);
-  void follow_large_array(uintptr_t addr, size_t size, bool finalizable);
-  void follow_array(uintptr_t addr, size_t size, bool finalizable);
+  bool is_array(zaddress addr) const;
+  void push_partial_array(zpointer* addr, size_t length, bool finalizable);
+  void follow_array_elements_small(zpointer* addr, size_t length, bool finalizable);
+  void follow_array_elements_large(zpointer* addr, size_t length, bool finalizable);
+  void follow_array_elements(zpointer* addr, size_t length, bool finalizable);
   void follow_partial_array(ZMarkStackEntry entry, bool finalizable);
   void follow_array_object(objArrayOop obj, bool finalizable);
   void follow_object(oop obj, bool finalizable);
   void mark_and_follow(ZMarkContext* context, ZMarkStackEntry entry);
 
-  template <typename T> bool drain(ZMarkContext* context, T* timeout);
+  bool rebalance_work(ZMarkContext* context);
+  bool drain(ZMarkContext* context);
   bool try_steal_local(ZMarkContext* context);
   bool try_steal_global(ZMarkContext* context);
   bool try_steal(ZMarkContext* context);
-  void idle() const;
-  bool flush(bool at_safepoint);
+  bool flush();
   bool try_proactive_flush();
-  bool try_flush(volatile size_t* nflush);
-  bool try_terminate();
-  bool try_complete();
+  bool try_terminate(ZMarkContext* context);
+  void leave();
   bool try_end();
 
-  void prepare_work();
-  void finish_work();
+  ZWorkers* workers() const;
 
-  void work_without_timeout(ZMarkContext* context);
-  void work_with_timeout(ZMarkContext* context, uint64_t timeout_in_micros);
-  void work(uint64_t timeout_in_micros);
+  bool follow_work(bool partial);
 
   void verify_all_stacks_empty() const;
+  void verify_worker_stacks_empty() const;
 
 public:
-  ZMark(ZWorkers* workers, ZPageTable* page_table);
+  ZMark(ZGeneration* generation, ZPageTable* page_table);
 
   bool is_initialized() const;
 
-  template <bool gc_thread, bool follow, bool finalizable, bool publish> void mark_object(uintptr_t addr);
+  template <bool resurrect, bool gc_thread, bool follow, bool finalizable>
+  void mark_object(zaddress addr);
 
   void start();
-  void mark(bool initial);
+  void mark_young_roots();
+  void mark_old_roots();
+  void mark_follow();
   bool end();
   void free();
 
   void flush_and_free();
   bool flush_and_free(Thread* thread);
+
+  // Following work
+  void prepare_work();
+  void finish_work();
+  void resize_workers(uint nworkers);
+  void follow_work_complete();
+  bool follow_work_partial();
+  bool try_terminate_flush();
 };
 
 #endif // SHARE_GC_Z_ZMARK_HPP
diff --git a/src/hotspot/share/gc/z/zMark.inline.hpp b/src/hotspot/share/gc/z/zMark.inline.hpp
index 289d8df7db4..b5302593610 100644
--- a/src/hotspot/share/gc/z/zMark.inline.hpp
+++ b/src/hotspot/share/gc/z/zMark.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,7 +27,9 @@
 #include "gc/z/zMark.hpp"
 
 #include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zMarkStack.inline.hpp"
+#include "gc/z/zMarkTerminate.inline.hpp"
 #include "gc/z/zPage.inline.hpp"
 #include "gc/z/zPageTable.inline.hpp"
 #include "gc/z/zThreadLocalData.hpp"
@@ -43,9 +45,9 @@
 // root processing has called ClassLoaderDataGraph::clear_claimed_marks(),
 // since it otherwise would interact badly with claiming of CLDs.
 
-template <bool gc_thread, bool follow, bool finalizable, bool publish>
-inline void ZMark::mark_object(uintptr_t addr) {
-  assert(ZAddress::is_marked(addr), "Should be marked");
+template <bool resurrect, bool gc_thread, bool follow, bool finalizable>
+inline void ZMark::mark_object(zaddress addr) {
+  assert(!ZVerifyOops || oopDesc::is_oop(to_oop(addr)), "Should be oop");
 
   ZPage* const page = _page_table->get(addr);
   if (page->is_allocating()) {
@@ -64,17 +66,25 @@ inline void ZMark::mark_object(uintptr_t addr) {
     }
   } else {
     // Don't push if already marked
-    if (page->is_object_marked<finalizable>(addr)) {
+    if (page->is_object_marked(addr, finalizable)) {
       // Already marked
       return;
     }
   }
 
+  if (resurrect) {
+    _terminate.set_resurrected(true);
+  }
+
   // Push
-  ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(Thread::current());
-  ZMarkStripe* const stripe = _stripes.stripe_for_addr(addr);
-  ZMarkStackEntry entry(addr, !mark_before_push, inc_live, follow, finalizable);
-  stacks->push(&_allocator, &_stripes, stripe, entry, publish);
+  ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::mark_stacks(Thread::current(), _generation->id());
+  ZMarkStripe* const stripe = _stripes.stripe_for_addr(untype(addr));
+  ZMarkStackEntry entry(untype(ZAddress::offset(addr)), !mark_before_push, inc_live, follow, finalizable);
+
+  assert(ZHeap::heap()->is_young(addr) == _generation->is_young(), "Phase/object mismatch");
+
+  const bool publish = !gc_thread;
+  stacks->push(&_allocator, &_stripes, stripe, &_terminate, entry, publish);
 }
 
 #endif // SHARE_GC_Z_ZMARK_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zMarkCache.cpp b/src/hotspot/share/gc/z/zMarkCache.cpp
index e41e3f3c3f4..f9b48417ac6 100644
--- a/src/hotspot/share/gc/z/zMarkCache.cpp
+++ b/src/hotspot/share/gc/z/zMarkCache.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,13 +26,17 @@
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/powerOfTwo.hpp"
 
+static size_t shift_for_stripes(size_t nstripes) {
+  return ZMarkStripeShift + exact_log2(nstripes);
+}
+
 ZMarkCacheEntry::ZMarkCacheEntry() :
-    _page(NULL),
+    _page(nullptr),
     _objects(0),
     _bytes(0) {}
 
 ZMarkCache::ZMarkCache(size_t nstripes) :
-    _shift(ZMarkStripeShift + exact_log2(nstripes)) {}
+    _shift(shift_for_stripes(nstripes)) {}
 
 ZMarkCache::~ZMarkCache() {
   // Evict all entries
@@ -40,3 +44,7 @@ ZMarkCache::~ZMarkCache() {
     _cache[i].evict();
   }
 }
+
+void ZMarkCache::set_nstripes(size_t nstripes) {
+  _shift = shift_for_stripes(nstripes);
+}
diff --git a/src/hotspot/share/gc/z/zMarkCache.hpp b/src/hotspot/share/gc/z/zMarkCache.hpp
index 8d902088c05..f866856100d 100644
--- a/src/hotspot/share/gc/z/zMarkCache.hpp
+++ b/src/hotspot/share/gc/z/zMarkCache.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -44,13 +44,15 @@ public:
 
 class ZMarkCache : public StackObj {
 private:
-  const size_t    _shift;
+  size_t          _shift;
   ZMarkCacheEntry _cache[ZMarkCacheSize];
 
 public:
   ZMarkCache(size_t nstripes);
   ~ZMarkCache();
 
+  void set_nstripes(size_t nstripes);
+
   void inc_live(ZPage* page, size_t bytes);
 };
 
diff --git a/src/hotspot/share/gc/z/zMarkCache.inline.hpp b/src/hotspot/share/gc/z/zMarkCache.inline.hpp
index ee67d333d59..de7203967a7 100644
--- a/src/hotspot/share/gc/z/zMarkCache.inline.hpp
+++ b/src/hotspot/share/gc/z/zMarkCache.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -43,16 +43,16 @@ inline void ZMarkCacheEntry::inc_live(ZPage* page, size_t bytes) {
 }
 
 inline void ZMarkCacheEntry::evict() {
-  if (_page != NULL) {
+  if (_page != nullptr) {
     // Write cached data out to page
     _page->inc_live(_objects, _bytes);
-    _page = NULL;
+    _page = nullptr;
   }
 }
 
 inline void ZMarkCache::inc_live(ZPage* page, size_t bytes) {
   const size_t mask = ZMarkCacheSize - 1;
-  const size_t index = (page->start() >> _shift) & mask;
+  const size_t index = (untype(page->start()) >> _shift) & mask;
   _cache[index].inc_live(page, bytes);
 }
 
diff --git a/src/hotspot/share/gc/z/zMarkContext.hpp b/src/hotspot/share/gc/z/zMarkContext.hpp
index 1b5ab76db7a..009252e524d 100644
--- a/src/hotspot/share/gc/z/zMarkContext.hpp
+++ b/src/hotspot/share/gc/z/zMarkContext.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,8 +34,9 @@ class ZMarkThreadLocalStacks;
 class ZMarkContext : public StackObj {
 private:
   ZMarkCache                    _cache;
-  ZMarkStripe* const            _stripe;
+  ZMarkStripe*                  _stripe;
   ZMarkThreadLocalStacks* const _stacks;
+  size_t                        _nstripes;
   StringDedup::Requests         _string_dedup_requests;
 
 public:
@@ -45,8 +46,12 @@ public:
 
   ZMarkCache* cache();
   ZMarkStripe* stripe();
+  void set_stripe(ZMarkStripe* stripe);
   ZMarkThreadLocalStacks* stacks();
   StringDedup::Requests* string_dedup_requests();
+
+  size_t nstripes();
+  void set_nstripes(size_t nstripes);
 };
 
 #endif // SHARE_GC_Z_ZMARKCONTEXT_HPP
diff --git a/src/hotspot/share/gc/z/zMarkContext.inline.hpp b/src/hotspot/share/gc/z/zMarkContext.inline.hpp
index b104ab61e4f..4a3237890fe 100644
--- a/src/hotspot/share/gc/z/zMarkContext.inline.hpp
+++ b/src/hotspot/share/gc/z/zMarkContext.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@ inline ZMarkContext::ZMarkContext(size_t nstripes,
     _cache(nstripes),
     _stripe(stripe),
     _stacks(stacks),
+    _nstripes(nstripes),
     _string_dedup_requests() {}
 
 inline ZMarkCache* ZMarkContext::cache() {
@@ -42,6 +43,10 @@ inline ZMarkStripe* ZMarkContext::stripe() {
   return _stripe;
 }
 
+inline void ZMarkContext::set_stripe(ZMarkStripe* stripe) {
+  _stripe = stripe;
+}
+
 inline ZMarkThreadLocalStacks* ZMarkContext::stacks() {
   return _stacks;
 }
@@ -50,4 +55,13 @@ inline StringDedup::Requests* ZMarkContext::string_dedup_requests() {
   return &_string_dedup_requests;
 }
 
+inline size_t ZMarkContext::nstripes() {
+  return _nstripes;
+}
+
+inline void ZMarkContext::set_nstripes(size_t nstripes) {
+  _cache.set_nstripes(nstripes);
+  _nstripes = nstripes;
+}
+
 #endif // SHARE_GC_Z_ZMARKCACHE_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zMarkStack.cpp b/src/hotspot/share/gc/z/zMarkStack.cpp
index b99c89a119a..64354fe022e 100644
--- a/src/hotspot/share/gc/z/zMarkStack.cpp
+++ b/src/hotspot/share/gc/z/zMarkStack.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,18 +24,25 @@
 #include "precompiled.hpp"
 #include "gc/z/zMarkStack.inline.hpp"
 #include "gc/z/zMarkStackAllocator.hpp"
+#include "gc/z/zMarkTerminate.inline.hpp"
 #include "logging/log.hpp"
+#include "runtime/atomic.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/powerOfTwo.hpp"
 
-ZMarkStripe::ZMarkStripe() :
-    _published(),
-    _overflowed() {}
+ZMarkStripe::ZMarkStripe(uintptr_t base) :
+    _published(base),
+    _overflowed(base) {}
 
-ZMarkStripeSet::ZMarkStripeSet() :
-    _nstripes(0),
+ZMarkStripeSet::ZMarkStripeSet(uintptr_t base) :
     _nstripes_mask(0),
-    _stripes() {}
+    _stripes() {
+
+  // Re-construct array elements with the correct base
+  for (size_t i = 0; i < ARRAY_SIZE(_stripes); i++) {
+    _stripes[i] = ZMarkStripe(base);
+  }
+}
 
 void ZMarkStripeSet::set_nstripes(size_t nstripes) {
   assert(is_power_of_2(nstripes), "Must be a power of two");
@@ -43,14 +50,19 @@ void ZMarkStripeSet::set_nstripes(size_t nstripes) {
   assert(nstripes >= 1, "Invalid number of stripes");
   assert(nstripes <= ZMarkStripesMax, "Invalid number of stripes");
 
-  _nstripes = nstripes;
-  _nstripes_mask = nstripes - 1;
+  // Mutators may read these values concurrently. It doesn't matter
+  // if they see the old or new values.
+  Atomic::store(&_nstripes_mask, nstripes - 1);
 
-  log_debug(gc, marking)("Using " SIZE_FORMAT " mark stripes", _nstripes);
+  log_debug(gc, marking)("Using " SIZE_FORMAT " mark stripes", nstripes);
+}
+
+size_t ZMarkStripeSet::nstripes() const {
+  return Atomic::load(&_nstripes_mask) + 1;
 }
 
 bool ZMarkStripeSet::is_empty() const {
-  for (size_t i = 0; i < _nstripes; i++) {
+  for (size_t i = 0; i < ZMarkStripesMax; i++) {
     if (!_stripes[i].is_empty()) {
       return false;
     }
@@ -60,35 +72,38 @@ bool ZMarkStripeSet::is_empty() const {
 }
 
 ZMarkStripe* ZMarkStripeSet::stripe_for_worker(uint nworkers, uint worker_id) {
-  const size_t spillover_limit = (nworkers / _nstripes) * _nstripes;
+  const size_t mask = Atomic::load(&_nstripes_mask);
+  const size_t nstripes = mask + 1;
+
+  const size_t spillover_limit = (nworkers / nstripes) * nstripes;
   size_t index;
 
   if (worker_id < spillover_limit) {
     // Not a spillover worker, use natural stripe
-    index = worker_id & _nstripes_mask;
+    index = worker_id & mask;
   } else {
     // Distribute spillover workers evenly across stripes
     const size_t spillover_nworkers = nworkers - spillover_limit;
     const size_t spillover_worker_id = worker_id - spillover_limit;
-    const double spillover_chunk = (double)_nstripes / (double)spillover_nworkers;
+    const double spillover_chunk = (double)nstripes / (double)spillover_nworkers;
     index = spillover_worker_id * spillover_chunk;
   }
 
-  assert(index < _nstripes, "Invalid index");
+  assert(index < nstripes, "Invalid index");
   return &_stripes[index];
 }
 
 ZMarkThreadLocalStacks::ZMarkThreadLocalStacks() :
-    _magazine(NULL) {
+    _magazine(nullptr) {
   for (size_t i = 0; i < ZMarkStripesMax; i++) {
-    _stacks[i] = NULL;
+    _stacks[i] = nullptr;
   }
 }
 
 bool ZMarkThreadLocalStacks::is_empty(const ZMarkStripeSet* stripes) const {
-  for (size_t i = 0; i < stripes->nstripes(); i++) {
+  for (size_t i = 0; i < ZMarkStripesMax; i++) {
     ZMarkStack* const stack = _stacks[i];
-    if (stack != NULL) {
+    if (stack != nullptr) {
       return false;
     }
   }
@@ -97,21 +112,21 @@ bool ZMarkThreadLocalStacks::is_empty(const ZMarkStripeSet* stripes) const {
 }
 
 ZMarkStack* ZMarkThreadLocalStacks::allocate_stack(ZMarkStackAllocator* allocator) {
-  if (_magazine == NULL) {
+  if (_magazine == nullptr) {
     // Allocate new magazine
     _magazine = allocator->alloc_magazine();
-    if (_magazine == NULL) {
-      return NULL;
+    if (_magazine == nullptr) {
+      return nullptr;
     }
   }
 
-  ZMarkStack* stack = NULL;
+  ZMarkStack* stack = nullptr;
 
   if (!_magazine->pop(stack)) {
     // Magazine is empty, convert magazine into a new stack
     _magazine->~ZMarkStackMagazine();
     stack = new ((void*)_magazine) ZMarkStack();
-    _magazine = NULL;
+    _magazine = nullptr;
   }
 
   return stack;
@@ -119,7 +134,7 @@ ZMarkStack* ZMarkThreadLocalStacks::allocate_stack(ZMarkStackAllocator* allocato
 
 void ZMarkThreadLocalStacks::free_stack(ZMarkStackAllocator* allocator, ZMarkStack* stack) {
   for (;;) {
-    if (_magazine == NULL) {
+    if (_magazine == nullptr) {
       // Convert stack into a new magazine
       stack->~ZMarkStack();
       _magazine = new ((void*)stack) ZMarkStackMagazine();
@@ -133,22 +148,23 @@ void ZMarkThreadLocalStacks::free_stack(ZMarkStackAllocator* allocator, ZMarkSta
 
     // Free and uninstall full magazine
     allocator->free_magazine(_magazine);
-    _magazine = NULL;
+    _magazine = nullptr;
   }
 }
 
 bool ZMarkThreadLocalStacks::push_slow(ZMarkStackAllocator* allocator,
                                        ZMarkStripe* stripe,
                                        ZMarkStack** stackp,
+                                       ZMarkTerminate* terminate,
                                        ZMarkStackEntry entry,
                                        bool publish) {
   ZMarkStack* stack = *stackp;
 
   for (;;) {
-    if (stack == NULL) {
+    if (stack == nullptr) {
       // Allocate and install new stack
       *stackp = stack = allocate_stack(allocator);
-      if (stack == NULL) {
+      if (stack == nullptr) {
         // Out of mark stack memory
         return false;
       }
@@ -160,8 +176,8 @@ bool ZMarkThreadLocalStacks::push_slow(ZMarkStackAllocator* allocator,
     }
 
     // Publish/Overflow and uninstall stack
-    stripe->publish_stack(stack, publish);
-    *stackp = stack = NULL;
+    stripe->publish_stack(stack, terminate, publish);
+    *stackp = stack = nullptr;
   }
 }
 
@@ -172,10 +188,10 @@ bool ZMarkThreadLocalStacks::pop_slow(ZMarkStackAllocator* allocator,
   ZMarkStack* stack = *stackp;
 
   for (;;) {
-    if (stack == NULL) {
+    if (stack == nullptr) {
       // Try steal and install stack
       *stackp = stack = stripe->steal_stack();
-      if (stack == NULL) {
+      if (stack == nullptr) {
         // Nothing to steal
         return false;
       }
@@ -188,19 +204,19 @@ bool ZMarkThreadLocalStacks::pop_slow(ZMarkStackAllocator* allocator,
 
     // Free and uninstall stack
     free_stack(allocator, stack);
-    *stackp = stack = NULL;
+    *stackp = stack = nullptr;
   }
 }
 
-bool ZMarkThreadLocalStacks::flush(ZMarkStackAllocator* allocator, ZMarkStripeSet* stripes) {
+bool ZMarkThreadLocalStacks::flush(ZMarkStackAllocator* allocator, ZMarkStripeSet* stripes, ZMarkTerminate* terminate) {
   bool flushed = false;
 
   // Flush all stacks
-  for (size_t i = 0; i < stripes->nstripes(); i++) {
+  for (size_t i = 0; i < ZMarkStripesMax; i++) {
     ZMarkStripe* const stripe = stripes->stripe_at(i);
     ZMarkStack** const stackp = &_stacks[i];
     ZMarkStack* const stack = *stackp;
-    if (stack == NULL) {
+    if (stack == nullptr) {
       continue;
     }
 
@@ -208,10 +224,10 @@ bool ZMarkThreadLocalStacks::flush(ZMarkStackAllocator* allocator, ZMarkStripeSe
     if (stack->is_empty()) {
       free_stack(allocator, stack);
     } else {
-      stripe->publish_stack(stack);
+      stripe->publish_stack(stack, terminate, true /* publish */);
       flushed = true;
     }
-    *stackp = NULL;
+    *stackp = nullptr;
   }
 
   return flushed;
@@ -219,8 +235,8 @@ bool ZMarkThreadLocalStacks::flush(ZMarkStackAllocator* allocator, ZMarkStripeSe
 
 void ZMarkThreadLocalStacks::free(ZMarkStackAllocator* allocator) {
   // Free and uninstall magazine
-  if (_magazine != NULL) {
+  if (_magazine != nullptr) {
     allocator->free_magazine(_magazine);
-    _magazine = NULL;
+    _magazine = nullptr;
   }
 }
diff --git a/src/hotspot/share/gc/z/zMarkStack.hpp b/src/hotspot/share/gc/z/zMarkStack.hpp
index 6c45d7ef7e0..5244830b3e2 100644
--- a/src/hotspot/share/gc/z/zMarkStack.hpp
+++ b/src/hotspot/share/gc/z/zMarkStack.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,8 @@
 #include "gc/z/zMarkStackEntry.hpp"
 #include "utilities/globalDefinitions.hpp"
 
+class ZMarkTerminate;
+
 template <typename T, size_t S>
 class ZStack {
 private:
@@ -52,13 +54,14 @@ public:
 template <typename T>
 class ZStackList {
 private:
+  uintptr_t   _base;
   T* volatile _head;
 
   T* encode_versioned_pointer(const T* stack, uint32_t version) const;
   void decode_versioned_pointer(const T* vstack, T** stack, uint32_t* version) const;
 
 public:
-  ZStackList();
+  explicit ZStackList(uintptr_t base);
 
   bool is_empty() const;
 
@@ -82,25 +85,24 @@ private:
   ZCACHE_ALIGNED ZMarkStackList _overflowed;
 
 public:
-  ZMarkStripe();
+  explicit ZMarkStripe(uintptr_t base = 0);
 
   bool is_empty() const;
 
-  void publish_stack(ZMarkStack* stack, bool publish = true);
+  void publish_stack(ZMarkStack* stack, ZMarkTerminate* terminate, bool publish);
   ZMarkStack* steal_stack();
 };
 
 class ZMarkStripeSet {
 private:
-  size_t      _nstripes;
   size_t      _nstripes_mask;
   ZMarkStripe _stripes[ZMarkStripesMax];
 
 public:
-  ZMarkStripeSet();
+  explicit ZMarkStripeSet(uintptr_t base);
 
-  size_t nstripes() const;
   void set_nstripes(size_t nstripes);
+  size_t nstripes() const;
 
   bool is_empty() const;
 
@@ -124,6 +126,7 @@ private:
   bool push_slow(ZMarkStackAllocator* allocator,
                  ZMarkStripe* stripe,
                  ZMarkStack** stackp,
+                 ZMarkTerminate* terminate,
                  ZMarkStackEntry entry,
                  bool publish);
 
@@ -147,6 +150,7 @@ public:
   bool push(ZMarkStackAllocator* allocator,
             ZMarkStripeSet* stripes,
             ZMarkStripe* stripe,
+            ZMarkTerminate* terminate,
             ZMarkStackEntry entry,
             bool publish);
 
@@ -156,7 +160,8 @@ public:
            ZMarkStackEntry& entry);
 
   bool flush(ZMarkStackAllocator* allocator,
-             ZMarkStripeSet* stripes);
+             ZMarkStripeSet* stripes,
+             ZMarkTerminate* terminate);
 
   void free(ZMarkStackAllocator* allocator);
 };
diff --git a/src/hotspot/share/gc/z/zMarkStack.inline.hpp b/src/hotspot/share/gc/z/zMarkStack.inline.hpp
index 5eaff5d7049..0c3329d9c07 100644
--- a/src/hotspot/share/gc/z/zMarkStack.inline.hpp
+++ b/src/hotspot/share/gc/z/zMarkStack.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,13 +26,14 @@
 
 #include "gc/z/zMarkStack.hpp"
 
-#include "utilities/debug.hpp"
+#include "gc/z/zMarkTerminate.inline.hpp"
 #include "runtime/atomic.hpp"
+#include "utilities/debug.hpp"
 
 template <typename T, size_t S>
 inline ZStack<T, S>::ZStack() :
     _top(0),
-    _next(NULL) {}
+    _next(nullptr) {}
 
 template <typename T, size_t S>
 inline bool ZStack<T, S>::is_empty() const {
@@ -75,17 +76,18 @@ inline ZStack<T, S>** ZStack<T, S>::next_addr() {
 }
 
 template <typename T>
-inline ZStackList<T>::ZStackList() :
-    _head(encode_versioned_pointer(NULL, 0)) {}
+inline ZStackList<T>::ZStackList(uintptr_t base) :
+    _base(base),
+    _head(encode_versioned_pointer(nullptr, 0)) {}
 
 template <typename T>
 inline T* ZStackList<T>::encode_versioned_pointer(const T* stack, uint32_t version) const {
   uint64_t addr;
 
-  if (stack == NULL) {
+  if (stack == nullptr) {
     addr = (uint32_t)-1;
   } else {
-    addr = ((uint64_t)stack - ZMarkStackSpaceStart) >> ZMarkStackSizeShift;
+    addr = ((uint64_t)stack - _base) >> ZMarkStackSizeShift;
   }
 
   return (T*)((addr << 32) | (uint64_t)version);
@@ -96,9 +98,9 @@ inline void ZStackList<T>::decode_versioned_pointer(const T* vstack, T** stack,
   const uint64_t addr = (uint64_t)vstack >> 32;
 
   if (addr == (uint32_t)-1) {
-    *stack = NULL;
+    *stack = nullptr;
   } else {
-    *stack = (T*)((addr << ZMarkStackSizeShift) + ZMarkStackSpaceStart);
+    *stack = (T*)((addr << ZMarkStackSizeShift) + _base);
   }
 
   *version = (uint32_t)(uint64_t)vstack;
@@ -107,11 +109,11 @@ inline void ZStackList<T>::decode_versioned_pointer(const T* vstack, T** stack,
 template <typename T>
 inline bool ZStackList<T>::is_empty() const {
   const T* vstack = _head;
-  T* stack = NULL;
+  T* stack = nullptr;
   uint32_t version = 0;
 
   decode_versioned_pointer(vstack, &stack, &version);
-  return stack == NULL;
+  return stack == nullptr;
 }
 
 template <typename T>
@@ -136,13 +138,13 @@ inline void ZStackList<T>::push(T* stack) {
 template <typename T>
 inline T* ZStackList<T>::pop() {
   T* vstack = _head;
-  T* stack = NULL;
+  T* stack = nullptr;
   uint32_t version = 0;
 
   for (;;) {
     decode_versioned_pointer(vstack, &stack, &version);
-    if (stack == NULL) {
-      return NULL;
+    if (stack == nullptr) {
+      return nullptr;
     }
 
     T* const new_vstack = encode_versioned_pointer(stack->next(), version + 1);
@@ -159,14 +161,14 @@ inline T* ZStackList<T>::pop() {
 
 template <typename T>
 inline void ZStackList<T>::clear() {
-  _head = encode_versioned_pointer(NULL, 0);
+  _head = encode_versioned_pointer(nullptr, 0);
 }
 
 inline bool ZMarkStripe::is_empty() const {
   return _published.is_empty() && _overflowed.is_empty();
 }
 
-inline void ZMarkStripe::publish_stack(ZMarkStack* stack, bool publish) {
+inline void ZMarkStripe::publish_stack(ZMarkStack* stack, ZMarkTerminate* terminate, bool publish) {
   // A stack is published either on the published list or the overflowed
   // list. The published list is used by mutators publishing stacks for GC
   // workers to work on, while the overflowed list is used by GC workers
@@ -178,42 +180,40 @@ inline void ZMarkStripe::publish_stack(ZMarkStack* stack, bool publish) {
   } else {
     _overflowed.push(stack);
   }
+
+  terminate->wake_up();
 }
 
 inline ZMarkStack* ZMarkStripe::steal_stack() {
   // Steal overflowed stacks first, then published stacks
   ZMarkStack* const stack = _overflowed.pop();
-  if (stack != NULL) {
+  if (stack != nullptr) {
     return stack;
   }
 
   return _published.pop();
 }
 
-inline size_t ZMarkStripeSet::nstripes() const {
-  return _nstripes;
-}
-
 inline size_t ZMarkStripeSet::stripe_id(const ZMarkStripe* stripe) const {
   const size_t index = ((uintptr_t)stripe - (uintptr_t)_stripes) / sizeof(ZMarkStripe);
-  assert(index < _nstripes, "Invalid index");
+  assert(index < ZMarkStripesMax, "Invalid index");
   return index;
 }
 
 inline ZMarkStripe* ZMarkStripeSet::stripe_at(size_t index) {
-  assert(index < _nstripes, "Invalid index");
+  assert(index < ZMarkStripesMax, "Invalid index");
   return &_stripes[index];
 }
 
 inline ZMarkStripe* ZMarkStripeSet::stripe_next(ZMarkStripe* stripe) {
-  const size_t index = (stripe_id(stripe) + 1) & _nstripes_mask;
-  assert(index < _nstripes, "Invalid index");
+  const size_t index = (stripe_id(stripe) + 1) & (ZMarkStripesMax - 1);
+  assert(index < ZMarkStripesMax, "Invalid index");
   return &_stripes[index];
 }
 
 inline ZMarkStripe* ZMarkStripeSet::stripe_for_addr(uintptr_t addr) {
-  const size_t index = (addr >> ZMarkStripeShift) & _nstripes_mask;
-  assert(index < _nstripes, "Invalid index");
+  const size_t index = (addr >> ZMarkStripeShift) & Atomic::load(&_nstripes_mask);
+  assert(index < ZMarkStripesMax, "Invalid index");
   return &_stripes[index];
 }
 
@@ -221,7 +221,7 @@ inline void ZMarkThreadLocalStacks::install(ZMarkStripeSet* stripes,
                                             ZMarkStripe* stripe,
                                             ZMarkStack* stack) {
   ZMarkStack** const stackp = &_stacks[stripes->stripe_id(stripe)];
-  assert(*stackp == NULL, "Should be empty");
+  assert(*stackp == nullptr, "Should be empty");
   *stackp = stack;
 }
 
@@ -229,8 +229,8 @@ inline ZMarkStack* ZMarkThreadLocalStacks::steal(ZMarkStripeSet* stripes,
                                                  ZMarkStripe* stripe) {
   ZMarkStack** const stackp = &_stacks[stripes->stripe_id(stripe)];
   ZMarkStack* const stack = *stackp;
-  if (stack != NULL) {
-    *stackp = NULL;
+  if (stack != nullptr) {
+    *stackp = nullptr;
   }
 
   return stack;
@@ -239,15 +239,16 @@ inline ZMarkStack* ZMarkThreadLocalStacks::steal(ZMarkStripeSet* stripes,
 inline bool ZMarkThreadLocalStacks::push(ZMarkStackAllocator* allocator,
                                          ZMarkStripeSet* stripes,
                                          ZMarkStripe* stripe,
+                                         ZMarkTerminate* terminate,
                                          ZMarkStackEntry entry,
                                          bool publish) {
   ZMarkStack** const stackp = &_stacks[stripes->stripe_id(stripe)];
   ZMarkStack* const stack = *stackp;
-  if (stack != NULL && stack->push(entry)) {
+  if (stack != nullptr && stack->push(entry)) {
     return true;
   }
 
-  return push_slow(allocator, stripe, stackp, entry, publish);
+  return push_slow(allocator, stripe, stackp, terminate, entry, publish);
 }
 
 inline bool ZMarkThreadLocalStacks::pop(ZMarkStackAllocator* allocator,
@@ -256,7 +257,7 @@ inline bool ZMarkThreadLocalStacks::pop(ZMarkStackAllocator* allocator,
                                         ZMarkStackEntry& entry) {
   ZMarkStack** const stackp = &_stacks[stripes->stripe_id(stripe)];
   ZMarkStack* const stack = *stackp;
-  if (stack != NULL && stack->pop(entry)) {
+  if (stack != nullptr && stack->pop(entry)) {
     return true;
   }
 
diff --git a/src/hotspot/share/gc/z/zMarkStackAllocator.cpp b/src/hotspot/share/gc/z/zMarkStackAllocator.cpp
index dc38b75f874..747e2f4e2ae 100644
--- a/src/hotspot/share/gc/z/zMarkStackAllocator.cpp
+++ b/src/hotspot/share/gc/z/zMarkStackAllocator.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,8 +32,6 @@
 #include "runtime/os.hpp"
 #include "utilities/debug.hpp"
 
-uintptr_t ZMarkStackSpaceStart;
-
 ZMarkStackSpace::ZMarkStackSpace() :
     _expand_lock(),
     _start(0),
@@ -52,9 +50,6 @@ ZMarkStackSpace::ZMarkStackSpace() :
   // Successfully initialized
   _start = _top = _end = addr;
 
-  // Register mark stack space start
-  ZMarkStackSpaceStart = _start;
-
   // Prime space
   _end += expand_space();
 }
@@ -63,6 +58,10 @@ bool ZMarkStackSpace::is_initialized() const {
   return _start != 0;
 }
 
+uintptr_t ZMarkStackSpace::start() const {
+  return _start;
+}
+
 size_t ZMarkStackSpace::size() const {
   return _end - _start;
 }
@@ -170,13 +169,18 @@ void ZMarkStackSpace::free() {
 }
 
 ZMarkStackAllocator::ZMarkStackAllocator() :
-    _freelist(),
-    _space() {}
+    _space(),
+    _freelist(_space.start()),
+    _expanded_recently(false) {}
 
 bool ZMarkStackAllocator::is_initialized() const {
   return _space.is_initialized();
 }
 
+uintptr_t ZMarkStackAllocator::start() const {
+  return _space.start();
+}
+
 size_t ZMarkStackAllocator::size() const {
   return _space.size();
 }
@@ -198,19 +202,31 @@ ZMarkStackMagazine* ZMarkStackAllocator::create_magazine_from_space(uintptr_t ad
 ZMarkStackMagazine* ZMarkStackAllocator::alloc_magazine() {
   // Try allocating from the free list first
   ZMarkStackMagazine* const magazine = _freelist.pop();
-  if (magazine != NULL) {
+  if (magazine != nullptr) {
     return magazine;
   }
 
+  if (!Atomic::load(&_expanded_recently)) {
+    Atomic::cmpxchg(&_expanded_recently, false, true);
+  }
+
   // Allocate new magazine
   const uintptr_t addr = _space.alloc(ZMarkStackMagazineSize);
   if (addr == 0) {
-    return NULL;
+    return nullptr;
   }
 
   return create_magazine_from_space(addr, ZMarkStackMagazineSize);
 }
 
+bool ZMarkStackAllocator::clear_and_get_expanded_recently() {
+  if (!Atomic::load(&_expanded_recently)) {
+    return false;
+  }
+
+  return Atomic::cmpxchg(&_expanded_recently, true, false);
+}
+
 void ZMarkStackAllocator::free_magazine(ZMarkStackMagazine* magazine) {
   _freelist.push(magazine);
 }
diff --git a/src/hotspot/share/gc/z/zMarkStackAllocator.hpp b/src/hotspot/share/gc/z/zMarkStackAllocator.hpp
index 68fd9d14338..dc18a23e101 100644
--- a/src/hotspot/share/gc/z/zMarkStackAllocator.hpp
+++ b/src/hotspot/share/gc/z/zMarkStackAllocator.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
 
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zLock.hpp"
+#include "gc/z/zMarkStack.hpp"
 #include "utilities/globalDefinitions.hpp"
 
 class ZMarkStackSpace {
@@ -34,6 +35,7 @@ private:
   uintptr_t          _start;
   volatile uintptr_t _top;
   volatile uintptr_t _end;
+  volatile bool      _recently_expanded;
 
   size_t used() const;
 
@@ -48,16 +50,18 @@ public:
 
   bool is_initialized() const;
 
+  uintptr_t start() const;
   size_t size() const;
 
   uintptr_t alloc(size_t size);
   void free();
 };
 
-class ZMarkStackAllocator {
+class ZMarkStackAllocator : public CHeapObj<mtGC> {
 private:
-  ZCACHE_ALIGNED ZMarkStackMagazineList _freelist;
   ZCACHE_ALIGNED ZMarkStackSpace        _space;
+  ZCACHE_ALIGNED ZMarkStackMagazineList _freelist;
+  ZCACHE_ALIGNED volatile bool          _expanded_recently;
 
   ZMarkStackMagazine* create_magazine_from_space(uintptr_t addr, size_t size);
 
@@ -66,8 +70,11 @@ public:
 
   bool is_initialized() const;
 
+  uintptr_t start() const;
   size_t size() const;
 
+  bool clear_and_get_expanded_recently();
+
   ZMarkStackMagazine* alloc_magazine();
   void free_magazine(ZMarkStackMagazine* magazine);
 
diff --git a/src/hotspot/share/gc/z/zMarkTerminate.hpp b/src/hotspot/share/gc/z/zMarkTerminate.hpp
index ca29566d2c1..cff1f8e73fa 100644
--- a/src/hotspot/share/gc/z/zMarkTerminate.hpp
+++ b/src/hotspot/share/gc/z/zMarkTerminate.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,31 +24,33 @@
 #ifndef SHARE_GC_Z_ZMARKTERMINATE_HPP
 #define SHARE_GC_Z_ZMARKTERMINATE_HPP
 
-#include "gc/z/zGlobals.hpp"
-#include "memory/allocation.hpp"
+#include "gc/z/zLock.hpp"
 #include "utilities/globalDefinitions.hpp"
 
+class ZMarkStripeSet;
+
 class ZMarkTerminate {
 private:
-  uint                         _nworkers;
-  ZCACHE_ALIGNED volatile uint _nworking_stage0;
-  volatile uint                _nworking_stage1;
+  uint           _nworkers;
+  volatile uint  _nworking;
+  volatile uint  _nawakening;
+  volatile bool  _resurrected;
+  ZConditionLock _lock;
 
-  bool enter_stage(volatile uint* nworking_stage);
-  void exit_stage(volatile uint* nworking_stage);
-  bool try_exit_stage(volatile uint* nworking_stage);
+  void maybe_reduce_stripes(ZMarkStripeSet* stripes, size_t used_nstripes);
 
 public:
   ZMarkTerminate();
 
   void reset(uint nworkers);
+  void leave();
 
-  bool enter_stage0();
-  void exit_stage0();
-  bool try_exit_stage0();
+  bool saturated() const;
 
-  bool enter_stage1();
-  bool try_exit_stage1();
+  void wake_up();
+  bool try_terminate(ZMarkStripeSet* stripes, size_t used_nstripes);
+  void set_resurrected(bool value);
+  bool resurrected() const;
 };
 
 #endif // SHARE_GC_Z_ZMARKTERMINATE_HPP
diff --git a/src/hotspot/share/gc/z/zMarkTerminate.inline.hpp b/src/hotspot/share/gc/z/zMarkTerminate.inline.hpp
index f738b65a1e3..49b9ac7aeaa 100644
--- a/src/hotspot/share/gc/z/zMarkTerminate.inline.hpp
+++ b/src/hotspot/share/gc/z/zMarkTerminate.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,63 +26,119 @@
 
 #include "gc/z/zMarkTerminate.hpp"
 
+#include "gc/shared/suspendibleThreadSet.hpp"
+#include "gc/z/zLock.inline.hpp"
+#include "logging/log.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/osThread.hpp"
+#include "runtime/thread.inline.hpp"
 
 inline ZMarkTerminate::ZMarkTerminate() :
     _nworkers(0),
-    _nworking_stage0(0),
-    _nworking_stage1(0) {}
+    _nworking(0),
+    _nawakening(0),
+    _resurrected(false),
+    _lock() {}
 
-inline bool ZMarkTerminate::enter_stage(volatile uint* nworking_stage) {
-  return Atomic::sub(nworking_stage, 1u) == 0;
+inline void ZMarkTerminate::reset(uint nworkers) {
+  Atomic::store(&_nworkers, nworkers);
+  Atomic::store(&_nworking, nworkers);
+  _nawakening = 0;
 }
 
-inline void ZMarkTerminate::exit_stage(volatile uint* nworking_stage) {
-  Atomic::add(nworking_stage, 1u);
-}
+inline void ZMarkTerminate::leave() {
+  SuspendibleThreadSetLeaver sts_leaver;
+  ZLocker<ZConditionLock> locker(&_lock);
 
-inline bool ZMarkTerminate::try_exit_stage(volatile uint* nworking_stage) {
-  uint nworking = Atomic::load(nworking_stage);
-
-  for (;;) {
-    if (nworking == 0) {
-      return false;
-    }
-
-    const uint new_nworking = nworking + 1;
-    const uint prev_nworking = Atomic::cmpxchg(nworking_stage, nworking, new_nworking);
-    if (prev_nworking == nworking) {
-      // Success
-      return true;
-    }
-
-    // Retry
-    nworking = prev_nworking;
+  Atomic::store(&_nworking, _nworking - 1);
+  if (_nworking == 0) {
+    // Last thread leaving; notify waiters
+    _lock.notify_all();
   }
 }
 
-inline void ZMarkTerminate::reset(uint nworkers) {
-  _nworkers = _nworking_stage0 = _nworking_stage1 = nworkers;
+inline void ZMarkTerminate::maybe_reduce_stripes(ZMarkStripeSet* stripes, size_t used_nstripes) {
+  size_t nstripes = stripes->nstripes();
+  if (used_nstripes == nstripes && nstripes > 1u) {
+    nstripes >>= 1;
+    stripes->set_nstripes(nstripes);
+  }
 }
 
-inline bool ZMarkTerminate::enter_stage0() {
-  return enter_stage(&_nworking_stage0);
+inline bool ZMarkTerminate::try_terminate(ZMarkStripeSet* stripes, size_t used_nstripes) {
+  SuspendibleThreadSetLeaver sts_leaver;
+  ZLocker<ZConditionLock> locker(&_lock);
+
+  Atomic::store(&_nworking, _nworking - 1);
+  if (_nworking == 0) {
+    // Last thread entering termination: success
+    _lock.notify_all();
+    return true;
+  }
+
+  // If a worker runs out of work, it might be a sign that we have too many stripes
+  // hiding work. Try to reduce the number of stripes if possible.
+  maybe_reduce_stripes(stripes, used_nstripes);
+  _lock.wait();
+
+  // We either got notification about more work
+  // or got a spurious wakeup; don't terminate
+  if (_nawakening > 0) {
+    Atomic::store(&_nawakening, _nawakening - 1);
+  }
+
+  if (_nworking == 0) {
+    // We got notified all work is done; terminate
+    return true;
+  }
+
+  Atomic::store(&_nworking, _nworking + 1);
+
+  return false;
 }
 
-inline void ZMarkTerminate::exit_stage0() {
-  exit_stage(&_nworking_stage0);
+inline void ZMarkTerminate::wake_up() {
+  uint nworking = Atomic::load(&_nworking);
+  uint nawakening = Atomic::load(&_nawakening);
+  if (nworking + nawakening == Atomic::load(&_nworkers)) {
+    // Everyone is working or about to
+    return;
+  }
+
+  if (nworking == 0) {
+    // Marking when marking task is not active
+    return;
+  }
+
+  ZLocker<ZConditionLock> locker(&_lock);
+  if (_nworking + _nawakening != _nworkers) {
+    // Everyone is not working
+    Atomic::store(&_nawakening, _nawakening + 1);
+    _lock.notify();
+  }
 }
 
-inline bool ZMarkTerminate::try_exit_stage0() {
-  return try_exit_stage(&_nworking_stage0);
+inline bool ZMarkTerminate::saturated() const {
+  uint nworking = Atomic::load(&_nworking);
+  uint nawakening = Atomic::load(&_nawakening);
+
+  return nworking + nawakening == Atomic::load(&_nworkers);
 }
 
-inline bool ZMarkTerminate::enter_stage1() {
-  return enter_stage(&_nworking_stage1);
+inline void ZMarkTerminate::set_resurrected(bool value) {
+  // Update resurrected if it changed
+  if (resurrected() != value) {
+    Atomic::store(&_resurrected, value);
+    if (value) {
+      log_debug(gc, marking)("Resurrection broke termination");
+    } else {
+      log_debug(gc, marking)("Try terminate after resurrection");
+    }
+  }
 }
 
-inline bool ZMarkTerminate::try_exit_stage1() {
-  return try_exit_stage(&_nworking_stage1);
+inline bool ZMarkTerminate::resurrected() const {
+  return Atomic::load(&_resurrected);
 }
 
 #endif // SHARE_GC_Z_ZMARKTERMINATE_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zMemory.cpp b/src/hotspot/share/gc/z/zMemory.cpp
index 7b1c0749f75..23d02892eb2 100644
--- a/src/hotspot/share/gc/z/zMemory.cpp
+++ b/src/hotspot/share/gc/z/zMemory.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,56 +26,56 @@
 #include "gc/z/zLock.inline.hpp"
 #include "gc/z/zMemory.inline.hpp"
 
-ZMemory* ZMemoryManager::create(uintptr_t start, size_t size) {
+ZMemory* ZMemoryManager::create(zoffset start, size_t size) {
   ZMemory* const area = new ZMemory(start, size);
-  if (_callbacks._create != NULL) {
+  if (_callbacks._create != nullptr) {
     _callbacks._create(area);
   }
   return area;
 }
 
 void ZMemoryManager::destroy(ZMemory* area) {
-  if (_callbacks._destroy != NULL) {
+  if (_callbacks._destroy != nullptr) {
     _callbacks._destroy(area);
   }
   delete area;
 }
 
 void ZMemoryManager::shrink_from_front(ZMemory* area, size_t size) {
-  if (_callbacks._shrink_from_front != NULL) {
+  if (_callbacks._shrink_from_front != nullptr) {
     _callbacks._shrink_from_front(area, size);
   }
   area->shrink_from_front(size);
 }
 
 void ZMemoryManager::shrink_from_back(ZMemory* area, size_t size) {
-  if (_callbacks._shrink_from_back != NULL) {
+  if (_callbacks._shrink_from_back != nullptr) {
     _callbacks._shrink_from_back(area, size);
   }
   area->shrink_from_back(size);
 }
 
 void ZMemoryManager::grow_from_front(ZMemory* area, size_t size) {
-  if (_callbacks._grow_from_front != NULL) {
+  if (_callbacks._grow_from_front != nullptr) {
     _callbacks._grow_from_front(area, size);
   }
   area->grow_from_front(size);
 }
 
 void ZMemoryManager::grow_from_back(ZMemory* area, size_t size) {
-  if (_callbacks._grow_from_back != NULL) {
+  if (_callbacks._grow_from_back != nullptr) {
     _callbacks._grow_from_back(area, size);
   }
   area->grow_from_back(size);
 }
 
 ZMemoryManager::Callbacks::Callbacks() :
-    _create(NULL),
-    _destroy(NULL),
-    _shrink_from_front(NULL),
-    _shrink_from_back(NULL),
-    _grow_from_front(NULL),
-    _grow_from_back(NULL) {}
+    _create(nullptr),
+    _destroy(nullptr),
+    _shrink_from_front(nullptr),
+    _shrink_from_back(nullptr),
+    _grow_from_front(nullptr),
+    _grow_from_back(nullptr) {}
 
 ZMemoryManager::ZMemoryManager() :
     _freelist(),
@@ -85,19 +85,19 @@ void ZMemoryManager::register_callbacks(const Callbacks& callbacks) {
   _callbacks = callbacks;
 }
 
-uintptr_t ZMemoryManager::peek_low_address() const {
+zoffset ZMemoryManager::peek_low_address() const {
   ZLocker<ZLock> locker(&_lock);
 
   const ZMemory* const area = _freelist.first();
-  if (area != NULL) {
+  if (area != nullptr) {
     return area->start();
   }
 
   // Out of memory
-  return UINTPTR_MAX;
+  return zoffset(UINTPTR_MAX);
 }
 
-uintptr_t ZMemoryManager::alloc_low_address(size_t size) {
+zoffset ZMemoryManager::alloc_low_address(size_t size) {
   ZLocker<ZLock> locker(&_lock);
 
   ZListIterator<ZMemory> iter(&_freelist);
@@ -105,13 +105,13 @@ uintptr_t ZMemoryManager::alloc_low_address(size_t size) {
     if (area->size() >= size) {
       if (area->size() == size) {
         // Exact match, remove area
-        const uintptr_t start = area->start();
+        const zoffset start = area->start();
         _freelist.remove(area);
         destroy(area);
         return start;
       } else {
         // Larger than requested, shrink area
-        const uintptr_t start = area->start();
+        const zoffset start = area->start();
         shrink_from_front(area, size);
         return start;
       }
@@ -119,24 +119,24 @@ uintptr_t ZMemoryManager::alloc_low_address(size_t size) {
   }
 
   // Out of memory
-  return UINTPTR_MAX;
+  return zoffset(UINTPTR_MAX);
 }
 
-uintptr_t ZMemoryManager::alloc_low_address_at_most(size_t size, size_t* allocated) {
+zoffset ZMemoryManager::alloc_low_address_at_most(size_t size, size_t* allocated) {
   ZLocker<ZLock> locker(&_lock);
 
-  ZMemory* area = _freelist.first();
-  if (area != NULL) {
+  ZMemory* const area = _freelist.first();
+  if (area != nullptr) {
     if (area->size() <= size) {
       // Smaller than or equal to requested, remove area
-      const uintptr_t start = area->start();
+      const zoffset start = area->start();
       *allocated = area->size();
       _freelist.remove(area);
       destroy(area);
       return start;
     } else {
       // Larger than requested, shrink area
-      const uintptr_t start = area->start();
+      const zoffset start = area->start();
       shrink_from_front(area, size);
       *allocated = size;
       return start;
@@ -145,10 +145,10 @@ uintptr_t ZMemoryManager::alloc_low_address_at_most(size_t size, size_t* allocat
 
   // Out of memory
   *allocated = 0;
-  return UINTPTR_MAX;
+  return zoffset(UINTPTR_MAX);
 }
 
-uintptr_t ZMemoryManager::alloc_high_address(size_t size) {
+zoffset ZMemoryManager::alloc_high_address(size_t size) {
   ZLocker<ZLock> locker(&_lock);
 
   ZListReverseIterator<ZMemory> iter(&_freelist);
@@ -156,25 +156,25 @@ uintptr_t ZMemoryManager::alloc_high_address(size_t size) {
     if (area->size() >= size) {
       if (area->size() == size) {
         // Exact match, remove area
-        const uintptr_t start = area->start();
+        const zoffset start = area->start();
         _freelist.remove(area);
         destroy(area);
         return start;
       } else {
         // Larger than requested, shrink area
         shrink_from_back(area, size);
-        return area->end();
+        return to_zoffset(area->end());
       }
     }
   }
 
   // Out of memory
-  return UINTPTR_MAX;
+  return zoffset(UINTPTR_MAX);
 }
 
-void ZMemoryManager::free(uintptr_t start, size_t size) {
-  assert(start != UINTPTR_MAX, "Invalid address");
-  const uintptr_t end = start + size;
+void ZMemoryManager::free(zoffset start, size_t size) {
+  assert(start != zoffset(UINTPTR_MAX), "Invalid address");
+  const zoffset_end end = to_zoffset_end(start, size);
 
   ZLocker<ZLock> locker(&_lock);
 
@@ -182,7 +182,7 @@ void ZMemoryManager::free(uintptr_t start, size_t size) {
   for (ZMemory* area; iter.next(&area);) {
     if (start < area->start()) {
       ZMemory* const prev = _freelist.prev(area);
-      if (prev != NULL && start == prev->end()) {
+      if (prev != nullptr && start == prev->end()) {
         if (end == area->start()) {
           // Merge with prev and current area
           grow_from_back(prev, size + area->size());
@@ -209,7 +209,7 @@ void ZMemoryManager::free(uintptr_t start, size_t size) {
 
   // Insert last
   ZMemory* const last = _freelist.last();
-  if (last != NULL && start == last->end()) {
+  if (last != nullptr && start == last->end()) {
     // Merge with last area
     grow_from_back(last, size);
   } else {
diff --git a/src/hotspot/share/gc/z/zMemory.hpp b/src/hotspot/share/gc/z/zMemory.hpp
index 85c4f5d232d..e75ac071d1d 100644
--- a/src/hotspot/share/gc/z/zMemory.hpp
+++ b/src/hotspot/share/gc/z/zMemory.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
 #ifndef SHARE_GC_Z_ZMEMORY_HPP
 #define SHARE_GC_Z_ZMEMORY_HPP
 
+#include "gc/z/zAddress.hpp"
 #include "gc/z/zList.hpp"
 #include "gc/z/zLock.hpp"
 #include "memory/allocation.hpp"
@@ -32,15 +33,15 @@ class ZMemory : public CHeapObj<mtGC> {
   friend class ZList<ZMemory>;
 
 private:
-  uintptr_t          _start;
-  uintptr_t          _end;
+  zoffset            _start;
+  zoffset_end        _end;
   ZListNode<ZMemory> _node;
 
 public:
-  ZMemory(uintptr_t start, size_t size);
+  ZMemory(zoffset start, size_t size);
 
-  uintptr_t start() const;
-  uintptr_t end() const;
+  zoffset start() const;
+  zoffset_end end() const;
   size_t size() const;
 
   void shrink_from_front(size_t size);
@@ -70,7 +71,7 @@ private:
   ZList<ZMemory> _freelist;
   Callbacks      _callbacks;
 
-  ZMemory* create(uintptr_t start, size_t size);
+  ZMemory* create(zoffset start, size_t size);
   void destroy(ZMemory* area);
   void shrink_from_front(ZMemory* area, size_t size);
   void shrink_from_back(ZMemory* area, size_t size);
@@ -82,12 +83,12 @@ public:
 
   void register_callbacks(const Callbacks& callbacks);
 
-  uintptr_t peek_low_address() const;
-  uintptr_t alloc_low_address(size_t size);
-  uintptr_t alloc_low_address_at_most(size_t size, size_t* allocated);
-  uintptr_t alloc_high_address(size_t size);
+  zoffset peek_low_address() const;
+  zoffset alloc_low_address(size_t size);
+  zoffset alloc_low_address_at_most(size_t size, size_t* allocated);
+  zoffset alloc_high_address(size_t size);
 
-  void free(uintptr_t start, size_t size);
+  void free(zoffset start, size_t size);
 };
 
 #endif // SHARE_GC_Z_ZMEMORY_HPP
diff --git a/src/hotspot/share/gc/z/zMemory.inline.hpp b/src/hotspot/share/gc/z/zMemory.inline.hpp
index 895e38375ec..19cccd3f6f5 100644
--- a/src/hotspot/share/gc/z/zMemory.inline.hpp
+++ b/src/hotspot/share/gc/z/zMemory.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,18 +26,19 @@
 
 #include "gc/z/zMemory.hpp"
 
+#include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zList.inline.hpp"
 #include "utilities/debug.hpp"
 
-inline ZMemory::ZMemory(uintptr_t start, size_t size) :
+inline ZMemory::ZMemory(zoffset start, size_t size) :
     _start(start),
-    _end(start + size) {}
+    _end(to_zoffset_end(start, size)) {}
 
-inline uintptr_t ZMemory::start() const {
+inline zoffset ZMemory::start() const {
   return _start;
 }
 
-inline uintptr_t ZMemory::end() const {
+inline zoffset_end ZMemory::end() const {
   return _end;
 }
 
@@ -56,7 +57,7 @@ inline void ZMemory::shrink_from_back(size_t size) {
 }
 
 inline void ZMemory::grow_from_front(size_t size) {
-  assert(start() >= size, "Too big");
+  assert(size_t(start()) >= size, "Too big");
   _start -= size;
 }
 
diff --git a/src/hotspot/share/gc/z/zNMethod.cpp b/src/hotspot/share/gc/z/zNMethod.cpp
index db4d3775d23..a851daba84b 100644
--- a/src/hotspot/share/gc/z/zNMethod.cpp
+++ b/src/hotspot/share/gc/z/zNMethod.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,25 +22,32 @@
  */
 
 #include "precompiled.hpp"
+#include "code/codeCache.hpp"
 #include "code/relocInfo.hpp"
 #include "code/nmethod.hpp"
 #include "code/icBuffer.hpp"
 #include "gc/shared/barrierSet.hpp"
 #include "gc/shared/barrierSetNMethod.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
+#include "gc/z/zAddress.hpp"
+#include "gc/z/zArray.inline.hpp"
 #include "gc/z/zBarrier.inline.hpp"
-#include "gc/z/zGlobals.hpp"
+#include "gc/z/zBarrierSet.hpp"
+#include "gc/z/zBarrierSetAssembler.hpp"
+#include "gc/z/zBarrierSetNMethod.hpp"
 #include "gc/z/zLock.inline.hpp"
 #include "gc/z/zNMethod.hpp"
 #include "gc/z/zNMethodData.hpp"
 #include "gc/z/zNMethodTable.hpp"
 #include "gc/z/zTask.hpp"
+#include "gc/z/zUncoloredRoot.inline.hpp"
 #include "gc/z/zWorkers.hpp"
 #include "logging/log.hpp"
 #include "memory/allocation.inline.hpp"
 #include "memory/iterator.hpp"
 #include "memory/resourceArea.hpp"
 #include "memory/universe.hpp"
+#include "oops/klass.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/continuation.hpp"
@@ -55,44 +62,42 @@ static void set_gc_data(nmethod* nm, ZNMethodData* data) {
 }
 
 void ZNMethod::attach_gc_data(nmethod* nm) {
-  GrowableArray<oop*> immediate_oops;
-  bool non_immediate_oops = false;
+  ZArray<ZNMethodDataBarrier> barriers;
+  ZArray<oop*> immediate_oops;
+  bool has_non_immediate_oops = false;
 
-  // Find all oop relocations
+  // Find all barrier and oop relocations
   RelocIterator iter(nm);
   while (iter.next()) {
-    if (iter.type() != relocInfo::oop_type) {
-      // Not an oop
-      continue;
-    }
+    if (iter.type() == relocInfo::barrier_type) {
+      // Barrier relocation
+      barrier_Relocation* const reloc = iter.barrier_reloc();
+      barriers.push({ reloc->addr(), reloc->format() });
+    } else if (iter.type() == relocInfo::oop_type) {
+      // Oop relocation
+      oop_Relocation* const reloc = iter.oop_reloc();
 
-    oop_Relocation* r = iter.oop_reloc();
-
-    if (!r->oop_is_immediate()) {
-      // Non-immediate oop found
-      non_immediate_oops = true;
-      continue;
-    }
-
-    if (r->oop_value() != NULL) {
-      // Non-NULL immediate oop found. NULL oops can safely be
-      // ignored since the method will be re-registered if they
-      // are later patched to be non-NULL.
-      immediate_oops.push(r->oop_addr());
+      if (!reloc->oop_is_immediate()) {
+        // Non-immediate oop found
+        has_non_immediate_oops = true;
+      } else if (reloc->oop_value() != nullptr) {
+        // Non-null immediate oop found. null oops can safely be
+        // ignored since the method will be re-registered if they
+        // are later patched to be non-null.
+        immediate_oops.push(reloc->oop_addr());
+      }
     }
   }
 
   // Attach GC data to nmethod
   ZNMethodData* data = gc_data(nm);
-  if (data == NULL) {
+  if (data == nullptr) {
     data = new ZNMethodData();
     set_gc_data(nm, data);
   }
 
-  // Attach oops in GC data
-  ZNMethodDataOops* const new_oops = ZNMethodDataOops::create(immediate_oops, non_immediate_oops);
-  ZNMethodDataOops* const old_oops = data->swap_oops(new_oops);
-  ZNMethodDataOops::destroy(old_oops);
+  // Attach barriers and oops to GC data
+  data->swap(&barriers, &immediate_oops, has_non_immediate_oops);
 }
 
 ZReentrantLock* ZNMethod::lock_for_nmethod(nmethod* nm) {
@@ -100,47 +105,55 @@ ZReentrantLock* ZNMethod::lock_for_nmethod(nmethod* nm) {
 }
 
 void ZNMethod::log_register(const nmethod* nm) {
-  LogTarget(Trace, gc, nmethod) log;
+  LogTarget(Debug, gc, nmethod) log;
   if (!log.is_enabled()) {
     return;
   }
 
-  const ZNMethodDataOops* const oops = gc_data(nm)->oops();
+  ResourceMark rm;
 
-  log.print("Register NMethod: %s.%s (" PTR_FORMAT "), "
-            "Compiler: %s, Oops: %d, ImmediateOops: " SIZE_FORMAT ", NonImmediateOops: %s",
+  const ZNMethodData* const data = gc_data(nm);
+
+  log.print("Register NMethod: %s.%s (" PTR_FORMAT ") [" PTR_FORMAT ", " PTR_FORMAT "] "
+            "Compiler: %s, Barriers: %d, Oops: %d, ImmediateOops: %d, NonImmediateOops: %s",
             nm->method()->method_holder()->external_name(),
             nm->method()->name()->as_C_string(),
             p2i(nm),
+            p2i(nm->code_begin()),
+            p2i(nm->code_end()),
             nm->compiler_name(),
+            data->barriers()->length(),
             nm->oops_count() - 1,
-            oops->immediates_count(),
-            oops->has_non_immediates() ? "Yes" : "No");
+            data->immediate_oops()->length(),
+            data->has_non_immediate_oops() ? "Yes" : "No");
 
-  LogTarget(Trace, gc, nmethod, oops) log_oops;
-  if (!log_oops.is_enabled()) {
-    return;
+  LogTarget(Trace, gc, nmethod, barrier) log_barriers;
+  if (log_barriers.is_enabled()) {
+    // Print nmethod barriers
+    ZArrayIterator<ZNMethodDataBarrier> iter(data->barriers());
+    for (ZNMethodDataBarrier b; iter.next(&b);) {
+      log_barriers.print("       Barrier: %d @ " PTR_FORMAT,
+                         b._reloc_format, p2i(b._reloc_addr));
+    }
   }
 
-  // Print nmethod oops table
-  {
+  LogTarget(Trace, gc, nmethod, oops) log_oops;
+  if (log_oops.is_enabled()) {
+    // Print nmethod oops table
     oop* const begin = nm->oops_begin();
     oop* const end = nm->oops_end();
     for (oop* p = begin; p < end; p++) {
       const oop o = Atomic::load(p); // C1 PatchingStub may replace it concurrently.
-      const char* external_name = (o == nullptr) ? "N/A" : o->klass()->external_name();
-      log_oops.print("           Oop[" SIZE_FORMAT "] " PTR_FORMAT " (%s)",
-                     (p - begin), p2i(o), external_name);
+      const char* const external_name = (o == nullptr) ? "N/A" : o->klass()->external_name();
+      log_oops.print("           Oop: " PTR_FORMAT " (%s)",
+                     p2i(o), external_name);
     }
-  }
 
-  // Print nmethod immediate oops
-  {
-    oop** const begin = oops->immediates_begin();
-    oop** const end = oops->immediates_end();
-    for (oop** p = begin; p < end; p++) {
-      log_oops.print("  ImmediateOop[" SIZE_FORMAT "] " PTR_FORMAT " @ " PTR_FORMAT " (%s)",
-                     (p - begin), p2i(**p), p2i(*p), (**p)->klass()->external_name());
+    // Print nmethod immediate oops
+    ZArrayIterator<oop*> iter(data->immediate_oops());
+    for (oop* p; iter.next(&p);) {
+      log_oops.print("  ImmediateOop: " PTR_FORMAT " @ " PTR_FORMAT " (%s)",
+                     p2i(*p), p2i(p), (*p)->klass()->external_name());
     }
   }
 }
@@ -151,20 +164,44 @@ void ZNMethod::log_unregister(const nmethod* nm) {
     return;
   }
 
-  log.print("Unregister NMethod: %s.%s (" PTR_FORMAT ")",
+  ResourceMark rm;
+
+  log.print("Unregister NMethod: %s.%s (" PTR_FORMAT ") [" PTR_FORMAT ", " PTR_FORMAT "] ",
             nm->method()->method_holder()->external_name(),
             nm->method()->name()->as_C_string(),
-            p2i(nm));
+            p2i(nm),
+            p2i(nm->code_begin()),
+            p2i(nm->code_end()));
+}
+
+void ZNMethod::log_purge(const nmethod* nm) {
+  LogTarget(Debug, gc, nmethod) log;
+  if (!log.is_enabled()) {
+    return;
+  }
+
+  ResourceMark rm;
+
+  log.print("Purge NMethod: %s.%s (" PTR_FORMAT ") [" PTR_FORMAT ", " PTR_FORMAT "] ",
+            nm->method()->method_holder()->external_name(),
+            nm->method()->name()->as_C_string(),
+            p2i(nm),
+            p2i(nm->code_begin()),
+            p2i(nm->code_end()));
 }
 
 void ZNMethod::register_nmethod(nmethod* nm) {
-  ResourceMark rm;
-
   // Create and attach gc data
   attach_gc_data(nm);
 
+  ZLocker<ZReentrantLock> locker(lock_for_nmethod(nm));
+
   log_register(nm);
 
+  // Patch nmethod barriers
+  nmethod_patch_barriers(nm);
+
+  // Register nmethod
   ZNMethodTable::register_nmethod(nm);
 
   // Disarm nmethod entry barrier
@@ -172,11 +209,13 @@ void ZNMethod::register_nmethod(nmethod* nm) {
 }
 
 void ZNMethod::unregister_nmethod(nmethod* nm) {
-  ResourceMark rm;
-
   log_unregister(nm);
 
   ZNMethodTable::unregister_nmethod(nm);
+}
+
+void ZNMethod::purge_nmethod(nmethod* nm) {
+  log_purge(nm);
 
   // Destroy GC data
   delete gc_data(nm);
@@ -202,8 +241,16 @@ void ZNMethod::set_guard_value(nmethod* nm, int value) {
   bs->set_guard_value(nm, value);
 }
 
+void ZNMethod::nmethod_patch_barriers(nmethod* nm) {
+  ZBarrierSetAssembler* const bs_asm = ZBarrierSet::assembler();
+  ZArrayIterator<ZNMethodDataBarrier> iter(gc_data(nm)->barriers());
+  for (ZNMethodDataBarrier barrier; iter.next(&barrier);) {
+    bs_asm->patch_barrier_relocation(barrier._reloc_addr, barrier._reloc_format);
+  }
+}
+
 void ZNMethod::nmethod_oops_do(nmethod* nm, OopClosure* cl) {
-  ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
+  ZLocker<ZReentrantLock> locker(lock_for_nmethod(nm));
   ZNMethod::nmethod_oops_do_inner(nm, cl);
 }
 
@@ -219,55 +266,69 @@ void ZNMethod::nmethod_oops_do_inner(nmethod* nm, OopClosure* cl) {
     }
   }
 
-  ZNMethodDataOops* const oops = gc_data(nm)->oops();
+  ZNMethodData* const data = gc_data(nm);
 
   // Process immediate oops
   {
-    oop** const begin = oops->immediates_begin();
-    oop** const end = oops->immediates_end();
-    for (oop** p = begin; p < end; p++) {
-      if (*p != Universe::non_oop_word()) {
-        cl->do_oop(*p);
+    ZArrayIterator<oop*> iter(data->immediate_oops());
+    for (oop* p; iter.next(&p);) {
+      if (!Universe::contains_non_oop_word(p)) {
+        cl->do_oop(p);
       }
     }
   }
 
   // Process non-immediate oops
-  if (oops->has_non_immediates()) {
+  if (data->has_non_immediate_oops()) {
     nm->fix_oop_relocations();
   }
 }
 
-class ZNMethodOopClosure : public OopClosure {
-public:
-  virtual void do_oop(oop* p) {
-    if (ZResurrection::is_blocked()) {
-      ZBarrier::keep_alive_barrier_on_phantom_root_oop_field(p);
-    } else {
-      ZBarrier::load_barrier_on_root_oop_field(p);
-    }
+void ZNMethod::nmethods_do_begin(bool secondary) {
+  ZNMethodTable::nmethods_do_begin(secondary);
+}
+
+void ZNMethod::nmethods_do_end(bool secondary) {
+  ZNMethodTable::nmethods_do_end(secondary);
+}
+
+void ZNMethod::nmethods_do(bool secondary, NMethodClosure* cl) {
+  ZNMethodTable::nmethods_do(secondary, cl);
+}
+
+uintptr_t ZNMethod::color(nmethod* nm) {
+  BarrierSetNMethod* bs_nm = BarrierSet::barrier_set()->barrier_set_nmethod();
+  // color is stored at low order bits of int; implicit conversion to uintptr_t is fine
+  return bs_nm->guard_value(nm);
+}
+
+oop ZNMethod::load_oop(oop* p, DecoratorSet decorators) {
+  assert((decorators & ON_WEAK_OOP_REF) == 0,
+         "nmethod oops have phantom strength, not weak");
+  nmethod* const nm = CodeCache::find_nmethod((void*)p);
+  if (!is_armed(nm)) {
+    // If the nmethod entry barrier isn't armed, then it has been applied
+    // already. The implication is that the contents of the memory location
+    // is already a valid oop, and the barrier would have kept it alive if
+    // necessary. Therefore, no action is required, and we are allowed to
+    // simply read the oop.
+    return *p;
   }
 
-  virtual void do_oop(narrowOop* p) {
-    ShouldNotReachHere();
+  const bool keep_alive = (decorators & ON_PHANTOM_OOP_REF) != 0 &&
+                          (decorators & AS_NO_KEEPALIVE) == 0;
+  ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
+
+  // Make a local root
+  zaddress_unsafe obj = *ZUncoloredRoot::cast(p);
+
+  if (keep_alive) {
+    ZUncoloredRoot::process(&obj, ZNMethod::color(nm));
+  } else {
+    ZUncoloredRoot::process_no_keepalive(&obj, ZNMethod::color(nm));
   }
-};
 
-void ZNMethod::nmethod_oops_barrier(nmethod* nm) {
-  ZNMethodOopClosure cl;
-  nmethod_oops_do_inner(nm, &cl);
-}
-
-void ZNMethod::nmethods_do_begin() {
-  ZNMethodTable::nmethods_do_begin();
-}
-
-void ZNMethod::nmethods_do_end() {
-  ZNMethodTable::nmethods_do_end();
-}
-
-void ZNMethod::nmethods_do(NMethodClosure* cl) {
-  ZNMethodTable::nmethods_do(cl);
+  return to_oop(safe(obj));
 }
 
 class ZNMethodUnlinkClosure : public NMethodClosure {
@@ -290,6 +351,10 @@ public:
     }
 
     if (nm->is_unloading()) {
+      // Unlink from the ZNMethodTable
+      ZNMethod::unregister_nmethod(nm);
+
+      // Shared unlink
       ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
       nm->unlink();
       return;
@@ -298,9 +363,25 @@ public:
     ZLocker<ZReentrantLock> locker(ZNMethod::lock_for_nmethod(nm));
 
     if (ZNMethod::is_armed(nm)) {
-      // Heal oops and arm phase invariantly
-      ZNMethod::nmethod_oops_barrier(nm);
-      ZNMethod::set_guard_value(nm, 0);
+      const uintptr_t prev_color = ZNMethod::color(nm);
+      assert(prev_color != ZPointerStoreGoodMask, "Potentially non-monotonic transition");
+
+      // Heal oops and potentially mark young objects if there is a concurrent young collection.
+      ZUncoloredRootProcessOopClosure cl(prev_color);
+      ZNMethod::nmethod_oops_do_inner(nm, &cl);
+
+      // Disarm for marking and relocation, but leave the remset bits so this isn't store good.
+      // This makes sure the mutator still takes a slow path to fill in the nmethod epoch for
+      // the sweeper, to track continuations, if they exist in the system.
+      const zpointer new_disarm_value_ptr = ZAddress::color(zaddress::null, ZPointerMarkGoodMask | ZPointerRememberedMask);
+
+      // The new disarm value is mark good, and hence never store good. Therefore, this operation
+      // never completely disarms the nmethod. Therefore, we don't need to patch barriers yet
+      // via ZNMethod::nmethod_patch_barriers.
+      ZNMethod::set_guard_value(nm, (int)untype(new_disarm_value_ptr));
+
+      log_trace(gc, nmethod)("nmethod: " PTR_FORMAT " visited by unlinking [" PTR_FORMAT " -> " PTR_FORMAT "]", p2i(nm), prev_color, untype(new_disarm_value_ptr));
+      assert(ZNMethod::is_armed(nm), "Must be considered armed");
     }
 
     // Clear compiled ICs and exception caches
@@ -324,16 +405,16 @@ public:
       ZTask("ZNMethodUnlinkTask"),
       _cl(unloading_occurred),
       _verifier(verifier) {
-    ZNMethodTable::nmethods_do_begin();
+    ZNMethodTable::nmethods_do_begin(false /* secondary */);
   }
 
   ~ZNMethodUnlinkTask() {
-    ZNMethodTable::nmethods_do_end();
+    ZNMethodTable::nmethods_do_end(false /* secondary */);
   }
 
   virtual void work() {
     ICRefillVerifierMark mark(_verifier);
-    ZNMethodTable::nmethods_do(&_cl);
+    ZNMethodTable::nmethods_do(false /* secondary */, &_cl);
   }
 
   bool success() const {
@@ -356,7 +437,7 @@ void ZNMethod::unlink(ZWorkers* workers, bool unloading_occurred) {
     // Cleaning failed because we ran out of transitional IC stubs,
     // so we have to refill and try again. Refilling requires taking
     // a safepoint, so we temporarily leave the suspendible thread set.
-    SuspendibleThreadSetLeaver sts;
+    SuspendibleThreadSetLeaver sts_leaver;
     InlineCacheBuffer::refill_ic_stubs();
   }
 }
diff --git a/src/hotspot/share/gc/z/zNMethod.hpp b/src/hotspot/share/gc/z/zNMethod.hpp
index 543a3a21732..1c6ef82328c 100644
--- a/src/hotspot/share/gc/z/zNMethod.hpp
+++ b/src/hotspot/share/gc/z/zNMethod.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,7 +24,10 @@
 #ifndef SHARE_GC_Z_ZNMETHOD_HPP
 #define SHARE_GC_Z_ZNMETHOD_HPP
 
-#include "memory/allStatic.hpp"
+#include "memory/allocation.hpp"
+#include "memory/iterator.hpp"
+#include "oops/accessDecorators.hpp"
+#include "oops/oopsHierarchy.hpp"
 
 class nmethod;
 class NMethodClosure;
@@ -37,10 +40,12 @@ private:
 
   static void log_register(const nmethod* nm);
   static void log_unregister(const nmethod* nm);
+  static void log_purge(const nmethod* nm);
 
 public:
   static void register_nmethod(nmethod* nm);
   static void unregister_nmethod(nmethod* nm);
+  static void purge_nmethod(nmethod* nm);
 
   static bool supports_entry_barrier(nmethod* nm);
 
@@ -48,19 +53,22 @@ public:
   static void disarm(nmethod* nm);
   static void set_guard_value(nmethod* nm, int value);
 
+  static void nmethod_patch_barriers(nmethod* nm);
+
   static void nmethod_oops_do(nmethod* nm, OopClosure* cl);
   static void nmethod_oops_do_inner(nmethod* nm, OopClosure* cl);
 
-  static void nmethod_oops_barrier(nmethod* nm);
-
-  static void nmethods_do_begin();
-  static void nmethods_do_end();
-  static void nmethods_do(NMethodClosure* cl);
+  static void nmethods_do_begin(bool secondary);
+  static void nmethods_do_end(bool secondary);
+  static void nmethods_do(bool secondary, NMethodClosure* cl);
 
   static ZReentrantLock* lock_for_nmethod(nmethod* nm);
 
   static void unlink(ZWorkers* workers, bool unloading_occurred);
   static void purge();
+
+  static uintptr_t color(nmethod* nm);
+  static oop load_oop(oop* p, DecoratorSet decorators);
 };
 
 #endif // SHARE_GC_Z_ZNMETHOD_HPP
diff --git a/src/hotspot/share/gc/z/zNMethodData.cpp b/src/hotspot/share/gc/z/zNMethodData.cpp
index c6efbfe6617..d19a7af4dda 100644
--- a/src/hotspot/share/gc/z/zNMethodData.cpp
+++ b/src/hotspot/share/gc/z/zNMethodData.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,67 +22,40 @@
  */
 
 #include "precompiled.hpp"
-#include "gc/z/zAttachedArray.inline.hpp"
 #include "gc/z/zLock.inline.hpp"
 #include "gc/z/zNMethodData.hpp"
-#include "memory/allocation.hpp"
-#include "runtime/atomic.hpp"
-#include "utilities/align.hpp"
 #include "utilities/debug.hpp"
-#include "utilities/growableArray.hpp"
-
-ZNMethodDataOops* ZNMethodDataOops::create(const GrowableArray<oop*>& immediates, bool has_non_immediates) {
-  return ::new (AttachedArray::alloc(immediates.length())) ZNMethodDataOops(immediates, has_non_immediates);
-}
-
-void ZNMethodDataOops::destroy(ZNMethodDataOops* oops) {
-  AttachedArray::free(oops);
-}
-
-ZNMethodDataOops::ZNMethodDataOops(const GrowableArray<oop*>& immediates, bool has_non_immediates) :
-    _immediates(immediates.length()),
-    _has_non_immediates(has_non_immediates) {
-  // Save all immediate oops
-  for (size_t i = 0; i < immediates_count(); i++) {
-    immediates_begin()[i] = immediates.at(int(i));
-  }
-}
-
-size_t ZNMethodDataOops::immediates_count() const {
-  return _immediates.length();
-}
-
-oop** ZNMethodDataOops::immediates_begin() const {
-  return _immediates(this);
-}
-
-oop** ZNMethodDataOops::immediates_end() const {
-  return immediates_begin() + immediates_count();
-}
-
-bool ZNMethodDataOops::has_non_immediates() const {
-  return _has_non_immediates;
-}
 
 ZNMethodData::ZNMethodData() :
     _lock(),
-    _oops(NULL) {}
-
-ZNMethodData::~ZNMethodData() {
-  ZNMethodDataOops::destroy(_oops);
-}
+    _barriers(),
+    _immediate_oops(),
+    _has_non_immediate_oops(false) {}
 
 ZReentrantLock* ZNMethodData::lock() {
   return &_lock;
 }
 
-ZNMethodDataOops* ZNMethodData::oops() const {
-  return Atomic::load_acquire(&_oops);
+const ZArray<ZNMethodDataBarrier>* ZNMethodData::barriers() const {
+  assert(_lock.is_owned(), "Should be owned");
+  return &_barriers;
 }
 
-ZNMethodDataOops* ZNMethodData::swap_oops(ZNMethodDataOops* new_oops) {
-  ZLocker<ZReentrantLock> locker(&_lock);
-  ZNMethodDataOops* const old_oops = _oops;
-  _oops = new_oops;
-  return old_oops;
+const ZArray<oop*>* ZNMethodData::immediate_oops() const {
+  assert(_lock.is_owned(), "Should be owned");
+  return &_immediate_oops;
+}
+
+bool ZNMethodData::has_non_immediate_oops() const {
+  assert(_lock.is_owned(), "Should be owned");
+  return _has_non_immediate_oops;
+}
+
+void ZNMethodData::swap(ZArray<ZNMethodDataBarrier>* barriers,
+                        ZArray<oop*>* immediate_oops,
+                        bool has_non_immediate_oops) {
+  ZLocker<ZReentrantLock> locker(&_lock);
+  _barriers.swap(barriers);
+  _immediate_oops.swap(immediate_oops);
+  _has_non_immediate_oops = has_non_immediate_oops;
 }
diff --git a/src/hotspot/share/gc/z/zNMethodData.hpp b/src/hotspot/share/gc/z/zNMethodData.hpp
index 7afd6010554..1b1ce077efc 100644
--- a/src/hotspot/share/gc/z/zNMethodData.hpp
+++ b/src/hotspot/share/gc/z/zNMethodData.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,48 +24,36 @@
 #ifndef SHARE_GC_Z_ZNMETHODDATA_HPP
 #define SHARE_GC_Z_ZNMETHODDATA_HPP
 
-#include "gc/z/zAttachedArray.hpp"
+#include "gc/z/zArray.hpp"
 #include "gc/z/zLock.hpp"
 #include "memory/allocation.hpp"
 #include "oops/oopsHierarchy.hpp"
 #include "utilities/globalDefinitions.hpp"
 
-class nmethod;
-template <typename T> class GrowableArray;
-
-class ZNMethodDataOops {
-private:
-  typedef ZAttachedArray<ZNMethodDataOops, oop*> AttachedArray;
-
-  const AttachedArray _immediates;
-  const bool          _has_non_immediates;
-
-  ZNMethodDataOops(const GrowableArray<oop*>& immediates, bool has_non_immediates);
-
-public:
-  static ZNMethodDataOops* create(const GrowableArray<oop*>& immediates, bool has_non_immediates);
-  static void destroy(ZNMethodDataOops* oops);
-
-  size_t immediates_count() const;
-  oop** immediates_begin() const;
-  oop** immediates_end() const;
-
-  bool has_non_immediates() const;
+struct ZNMethodDataBarrier {
+  address _reloc_addr;
+  int     _reloc_format;
 };
 
 class ZNMethodData : public CHeapObj<mtGC> {
 private:
-  ZReentrantLock             _lock;
-  ZNMethodDataOops* volatile _oops;
+  ZReentrantLock              _lock;
+  ZArray<ZNMethodDataBarrier> _barriers;
+  ZArray<oop*>                _immediate_oops;
+  bool                        _has_non_immediate_oops;
 
 public:
   ZNMethodData();
-  ~ZNMethodData();
 
   ZReentrantLock* lock();
 
-  ZNMethodDataOops* oops() const;
-  ZNMethodDataOops* swap_oops(ZNMethodDataOops* oops);
+  const ZArray<ZNMethodDataBarrier>* barriers() const;
+  const ZArray<oop*>* immediate_oops() const;
+  bool has_non_immediate_oops() const;
+
+  void swap(ZArray<ZNMethodDataBarrier>* barriers,
+            ZArray<oop*>* immediate_oops,
+            bool has_non_immediate_oops);
 };
 
 #endif // SHARE_GC_Z_ZNMETHODDATA_HPP
diff --git a/src/hotspot/share/gc/z/zNMethodTable.cpp b/src/hotspot/share/gc/z/zNMethodTable.cpp
index 4dde10d15d3..f75af7af616 100644
--- a/src/hotspot/share/gc/z/zNMethodTable.cpp
+++ b/src/hotspot/share/gc/z/zNMethodTable.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,7 +27,6 @@
 #include "code/icBuffer.hpp"
 #include "gc/shared/barrierSet.hpp"
 #include "gc/shared/barrierSetNMethod.hpp"
-#include "gc/z/zGlobals.hpp"
 #include "gc/z/zHash.inline.hpp"
 #include "gc/z/zLock.inline.hpp"
 #include "gc/z/zNMethodData.hpp"
@@ -45,12 +44,13 @@
 #include "utilities/debug.hpp"
 #include "utilities/powerOfTwo.hpp"
 
-ZNMethodTableEntry* ZNMethodTable::_table = NULL;
+ZNMethodTableEntry* ZNMethodTable::_table = nullptr;
 size_t ZNMethodTable::_size = 0;
 size_t ZNMethodTable::_nregistered = 0;
 size_t ZNMethodTable::_nunregistered = 0;
 ZNMethodTableIteration ZNMethodTable::_iteration;
-ZSafeDeleteNoLock<ZNMethodTableEntry[]> ZNMethodTable::_safe_delete;
+ZNMethodTableIteration ZNMethodTable::_iteration_secondary;
+ZSafeDelete<ZNMethodTableEntry[]> ZNMethodTable::_safe_delete(false /* locked */);
 
 size_t ZNMethodTable::first_index(const nmethod* nm, size_t size) {
   assert(is_power_of_2(size), "Invalid size");
@@ -130,7 +130,7 @@ void ZNMethodTable::rebuild(size_t new_size) {
   }
 
   // Free old table
-  _safe_delete(_table);
+  _safe_delete.schedule_delete(_table);
 
   // Install new table
   _table = new_table;
@@ -167,6 +167,12 @@ void ZNMethodTable::rebuild_if_needed() {
   }
 }
 
+ZNMethodTableIteration* ZNMethodTable::iteration(bool secondary) {
+  return secondary
+      ? &_iteration_secondary
+      : &_iteration;
+}
+
 size_t ZNMethodTable::registered_nmethods() {
   return _nregistered;
 }
@@ -193,13 +199,13 @@ void ZNMethodTable::register_nmethod(nmethod* nm) {
 void ZNMethodTable::wait_until_iteration_done() {
   assert(CodeCache_lock->owned_by_self(), "Lock must be held");
 
-  while (_iteration.in_progress()) {
+  while (_iteration.in_progress() || _iteration_secondary.in_progress()) {
     CodeCache_lock->wait_without_safepoint_check();
   }
 }
 
 void ZNMethodTable::unregister_nmethod(nmethod* nm) {
-  assert(CodeCache_lock->owned_by_self(), "Lock must be held");
+  MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
 
   // Remove entry
   unregister_entry(_table, _size, nm);
@@ -207,21 +213,21 @@ void ZNMethodTable::unregister_nmethod(nmethod* nm) {
   _nregistered--;
 }
 
-void ZNMethodTable::nmethods_do_begin() {
+void ZNMethodTable::nmethods_do_begin(bool secondary) {
   MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
 
   // Do not allow the table to be deleted while iterating
   _safe_delete.enable_deferred_delete();
 
   // Prepare iteration
-  _iteration.nmethods_do_begin(_table, _size);
+  iteration(secondary)->nmethods_do_begin(_table, _size);
 }
 
-void ZNMethodTable::nmethods_do_end() {
+void ZNMethodTable::nmethods_do_end(bool secondary) {
   MutexLocker mu(CodeCache_lock, Mutex::_no_safepoint_check_flag);
 
   // Finish iteration
-  _iteration.nmethods_do_end();
+  iteration(secondary)->nmethods_do_end();
 
   // Allow the table to be deleted
   _safe_delete.disable_deferred_delete();
@@ -230,6 +236,6 @@ void ZNMethodTable::nmethods_do_end() {
   CodeCache_lock->notify_all();
 }
 
-void ZNMethodTable::nmethods_do(NMethodClosure* cl) {
-  _iteration.nmethods_do(cl);
+void ZNMethodTable::nmethods_do(bool secondary, NMethodClosure* cl) {
+  iteration(secondary)->nmethods_do(cl);
 }
diff --git a/src/hotspot/share/gc/z/zNMethodTable.hpp b/src/hotspot/share/gc/z/zNMethodTable.hpp
index a1af8512f69..e160ac1b39a 100644
--- a/src/hotspot/share/gc/z/zNMethodTable.hpp
+++ b/src/hotspot/share/gc/z/zNMethodTable.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -35,12 +35,13 @@ class ZWorkers;
 
 class ZNMethodTable : public AllStatic {
 private:
-  static ZNMethodTableEntry*                     _table;
-  static size_t                                  _size;
-  static size_t                                  _nregistered;
-  static size_t                                  _nunregistered;
-  static ZNMethodTableIteration                  _iteration;
-  static ZSafeDeleteNoLock<ZNMethodTableEntry[]> _safe_delete;
+  static ZNMethodTableEntry*               _table;
+  static size_t                            _size;
+  static size_t                            _nregistered;
+  static size_t                            _nunregistered;
+  static ZNMethodTableIteration            _iteration;
+  static ZNMethodTableIteration            _iteration_secondary;
+  static ZSafeDelete<ZNMethodTableEntry[]> _safe_delete;
 
   static ZNMethodTableEntry* create(size_t size);
   static void destroy(ZNMethodTableEntry* table);
@@ -54,6 +55,8 @@ private:
   static void rebuild(size_t new_size);
   static void rebuild_if_needed();
 
+  static ZNMethodTableIteration* iteration(bool secondary);
+
 public:
   static size_t registered_nmethods();
   static size_t unregistered_nmethods();
@@ -63,9 +66,9 @@ public:
 
   static void wait_until_iteration_done();
 
-  static void nmethods_do_begin();
-  static void nmethods_do_end();
-  static void nmethods_do(NMethodClosure* cl);
+  static void nmethods_do_begin(bool secondary);
+  static void nmethods_do_end(bool secondary);
+  static void nmethods_do(bool secondary, NMethodClosure* cl);
 
   static void unlink(ZWorkers* workers, bool unloading_occurred);
   static void purge(ZWorkers* workers);
diff --git a/src/hotspot/share/gc/z/zNMethodTableEntry.hpp b/src/hotspot/share/gc/z/zNMethodTableEntry.hpp
index 8b30542999d..1afb634faeb 100644
--- a/src/hotspot/share/gc/z/zNMethodTableEntry.hpp
+++ b/src/hotspot/share/gc/z/zNMethodTableEntry.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -58,7 +58,7 @@ public:
   explicit ZNMethodTableEntry(bool unregistered = false) :
       _entry(field_registered::encode(false) |
              field_unregistered::encode(unregistered) |
-             field_method::encode(NULL)) {}
+             field_method::encode(nullptr)) {}
 
   explicit ZNMethodTableEntry(nmethod* method) :
       _entry(field_registered::encode(true) |
diff --git a/src/hotspot/share/gc/z/zNMethodTableIteration.cpp b/src/hotspot/share/gc/z/zNMethodTableIteration.cpp
index b3f9471bad7..0c8fcf059dc 100644
--- a/src/hotspot/share/gc/z/zNMethodTableIteration.cpp
+++ b/src/hotspot/share/gc/z/zNMethodTableIteration.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,12 +30,12 @@
 #include "utilities/globalDefinitions.hpp"
 
 ZNMethodTableIteration::ZNMethodTableIteration() :
-    _table(NULL),
+    _table(nullptr),
     _size(0),
     _claimed(0) {}
 
 bool ZNMethodTableIteration::in_progress() const {
-  return _table != NULL;
+  return _table != nullptr;
 }
 
 void ZNMethodTableIteration::nmethods_do_begin(ZNMethodTableEntry* table, size_t size) {
@@ -50,7 +50,7 @@ void ZNMethodTableIteration::nmethods_do_end() {
   assert(_claimed >= _size, "Failed to claim all table entries");
 
   // Finish iteration
-  _table = NULL;
+  _table = nullptr;
 }
 
 void ZNMethodTableIteration::nmethods_do(NMethodClosure* cl) {
diff --git a/src/hotspot/share/gc/z/zObjArrayAllocator.cpp b/src/hotspot/share/gc/z/zObjArrayAllocator.cpp
index 391ba264a74..14b2045f36e 100644
--- a/src/hotspot/share/gc/z/zObjArrayAllocator.cpp
+++ b/src/hotspot/share/gc/z/zObjArrayAllocator.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -63,31 +63,79 @@ oop ZObjArrayAllocator::initialize(HeapWord* mem) const {
   // The array is going to be exposed before it has been completely
   // cleared, therefore we can't expose the header at the end of this
   // function. Instead explicitly initialize it according to our needs.
-  arrayOopDesc::set_mark(mem, markWord::prototype());
+
+  // Signal to the ZIterator that this is an invisible root, by setting
+  // the mark word to "marked". Reset to prototype() after the clearing.
+  arrayOopDesc::set_mark(mem, markWord::prototype().set_marked());
   arrayOopDesc::release_set_klass(mem, _klass);
   assert(_length >= 0, "length should be non-negative");
   arrayOopDesc::set_length(mem, _length);
 
   // Keep the array alive across safepoints through an invisible
-  // root. Invisible roots are not visited by the heap itarator
+  // root. Invisible roots are not visited by the heap iterator
   // and the marking logic will not attempt to follow its elements.
-  // Relocation knows how to dodge iterating over such objects.
-  ZThreadLocalData::set_invisible_root(_thread, (oop*)&mem);
+  // Relocation and remembered set code know how to dodge iterating
+  // over such objects.
+  ZThreadLocalData::set_invisible_root(_thread, (zaddress_unsafe*)&mem);
 
-  for (size_t processed = 0; processed < payload_size; processed += segment_max) {
-    // Calculate segment
-    HeapWord* const start = (HeapWord*)(mem + header + processed);
-    const size_t remaining = payload_size - processed;
-    const size_t segment_size = MIN2(remaining, segment_max);
+  uint32_t old_seqnum_before = ZGeneration::old()->seqnum();
+  uint32_t young_seqnum_before = ZGeneration::young()->seqnum();
+  uintptr_t color_before = ZPointerStoreGoodMask;
+  auto gc_safepoint_happened = [&]() {
+    return old_seqnum_before != ZGeneration::old()->seqnum() ||
+           young_seqnum_before != ZGeneration::young()->seqnum() ||
+           color_before != ZPointerStoreGoodMask;
+  };
 
-    // Clear segment
-    Copy::zero_to_words(start, segment_size);
+  bool seen_gc_safepoint = false;
 
-    // Safepoint
-    yield_for_safepoint();
+  auto initialize_memory = [&]() {
+    for (size_t processed = 0; processed < payload_size; processed += segment_max) {
+      // Clear segment
+      uintptr_t* const start = (uintptr_t*)(mem + header + processed);
+      const size_t remaining = payload_size - processed;
+      const size_t segment = MIN2(remaining, segment_max);
+      // Usually, the young marking code has the responsibility to color
+      // raw nulls, before they end up in the old generation. However, the
+      // invisible roots are hidden from the marking code, and therefore
+      // we must color the nulls already here in the initialization. The
+      // color we choose must be store bad for any subsequent stores, regardless
+      // of how many GC flips later it will arrive. That's why we OR in 11
+      // (ZPointerRememberedMask) in the remembered bits, similar to how
+      // forgotten old oops also have 11, for the very same reason.
+      // However, we opportunistically try to color without the 11 remembered
+      // bits, hoping to not get interrupted in the middle of a GC safepoint.
+      // Most of the time, we manage to do that, and can the avoid having GC
+      // barriers trigger slow paths for this.
+      const uintptr_t colored_null = seen_gc_safepoint ? (ZPointerStoreGoodMask | ZPointerRememberedMask)
+                                                       : ZPointerStoreGoodMask;
+      const uintptr_t fill_value = is_reference_type(element_type) ? colored_null : 0;
+      ZUtils::fill(start, segment, fill_value);
+
+      // Safepoint
+      yield_for_safepoint();
+
+      // Deal with safepoints
+      if (!seen_gc_safepoint && gc_safepoint_happened()) {
+        // The first time we observe a GC safepoint in the yield point,
+        // we have to restart processing with 11 remembered bits.
+        seen_gc_safepoint = true;
+        return false;
+      }
+    }
+    return true;
+  };
+
+  if (!initialize_memory()) {
+    // Re-color with 11 remset bits if we got intercepted by a GC safepoint
+    const bool result = initialize_memory();
+    assert(result, "Array initialization should always succeed the second time");
   }
 
   ZThreadLocalData::clear_invisible_root(_thread);
 
+  // Signal to the ZIterator that this is no longer an invisible root
+  oopDesc::release_set_mark(mem, markWord::prototype());
+
   return cast_to_oop(mem);
 }
diff --git a/src/hotspot/share/gc/z/zObjectAllocator.cpp b/src/hotspot/share/gc/z/zObjectAllocator.cpp
index bbed59c1301..746906cfd93 100644
--- a/src/hotspot/share/gc/z/zObjectAllocator.cpp
+++ b/src/hotspot/share/gc/z/zObjectAllocator.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,25 +29,24 @@
 #include "gc/z/zPage.inline.hpp"
 #include "gc/z/zPageTable.inline.hpp"
 #include "gc/z/zStat.hpp"
-#include "gc/z/zThread.inline.hpp"
 #include "gc/z/zValue.inline.hpp"
 #include "logging/log.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/safepoint.hpp"
+#include "runtime/thread.hpp"
 #include "utilities/align.hpp"
 #include "utilities/debug.hpp"
 
 static const ZStatCounter ZCounterUndoObjectAllocationSucceeded("Memory", "Undo Object Allocation Succeeded", ZStatUnitOpsPerSecond);
 static const ZStatCounter ZCounterUndoObjectAllocationFailed("Memory", "Undo Object Allocation Failed", ZStatUnitOpsPerSecond);
 
-ZObjectAllocator::ZObjectAllocator() :
+ZObjectAllocator::ZObjectAllocator(ZPageAge age) :
+    _age(age),
     _use_per_cpu_shared_small_pages(ZHeuristics::use_per_cpu_shared_small_pages()),
     _used(0),
     _undone(0),
-    _alloc_for_relocation(0),
-    _undo_alloc_for_relocation(0),
-    _shared_medium_page(NULL),
-    _shared_small_page(NULL) {}
+    _shared_medium_page(nullptr),
+    _shared_small_page(nullptr) {}
 
 ZPage** ZObjectAllocator::shared_small_page_addr() {
   return _use_per_cpu_shared_small_pages ? _shared_small_page.addr() : _shared_small_page.addr(0);
@@ -57,20 +56,9 @@ ZPage* const* ZObjectAllocator::shared_small_page_addr() const {
   return _use_per_cpu_shared_small_pages ? _shared_small_page.addr() : _shared_small_page.addr(0);
 }
 
-void ZObjectAllocator::register_alloc_for_relocation(const ZPageTable* page_table, uintptr_t addr, size_t size) {
-  const ZPage* const page = page_table->get(addr);
-  const size_t aligned_size = align_up(size, page->object_alignment());
-  Atomic::add(_alloc_for_relocation.addr(), aligned_size);
-}
-
-void ZObjectAllocator::register_undo_alloc_for_relocation(const ZPage* page, size_t size) {
-  const size_t aligned_size = align_up(size, page->object_alignment());
-  Atomic::add(_undo_alloc_for_relocation.addr(), aligned_size);
-}
-
-ZPage* ZObjectAllocator::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) {
-  ZPage* const page = ZHeap::heap()->alloc_page(type, size, flags);
-  if (page != NULL) {
+ZPage* ZObjectAllocator::alloc_page(ZPageType type, size_t size, ZAllocationFlags flags) {
+  ZPage* const page = ZHeap::heap()->alloc_page(type, size, flags, _age);
+  if (page != nullptr) {
     // Increment used bytes
     Atomic::add(_used.addr(), size);
   }
@@ -78,6 +66,10 @@ ZPage* ZObjectAllocator::alloc_page(uint8_t type, size_t size, ZAllocationFlags
   return page;
 }
 
+ZPage* ZObjectAllocator::alloc_page_for_relocation(ZPageType type, size_t size, ZAllocationFlags flags) {
+  return ZHeap::heap()->alloc_page(type, size, flags, _age);
+}
+
 void ZObjectAllocator::undo_alloc_page(ZPage* page) {
   // Increment undone bytes
   Atomic::add(_undone.addr(), page->size());
@@ -85,22 +77,22 @@ void ZObjectAllocator::undo_alloc_page(ZPage* page) {
   ZHeap::heap()->undo_alloc_page(page);
 }
 
-uintptr_t ZObjectAllocator::alloc_object_in_shared_page(ZPage** shared_page,
-                                                        uint8_t page_type,
-                                                        size_t page_size,
-                                                        size_t size,
-                                                        ZAllocationFlags flags) {
-  uintptr_t addr = 0;
+zaddress ZObjectAllocator::alloc_object_in_shared_page(ZPage** shared_page,
+                                                       ZPageType page_type,
+                                                       size_t page_size,
+                                                       size_t size,
+                                                       ZAllocationFlags flags) {
+  zaddress addr = zaddress::null;
   ZPage* page = Atomic::load_acquire(shared_page);
 
-  if (page != NULL) {
+  if (page != nullptr) {
     addr = page->alloc_object_atomic(size);
   }
 
-  if (addr == 0) {
+  if (is_null(addr)) {
     // Allocate new page
     ZPage* const new_page = alloc_page(page_type, page_size, flags);
-    if (new_page != NULL) {
+    if (new_page != nullptr) {
       // Allocate object before installing the new page
       addr = new_page->alloc_object(size);
 
@@ -108,15 +100,15 @@ uintptr_t ZObjectAllocator::alloc_object_in_shared_page(ZPage** shared_page,
       // Install new page
       ZPage* const prev_page = Atomic::cmpxchg(shared_page, page, new_page);
       if (prev_page != page) {
-        if (prev_page == NULL) {
+        if (prev_page == nullptr) {
           // Previous page was retired, retry installing the new page
           page = prev_page;
           goto retry;
         }
 
         // Another page already installed, try allocation there first
-        const uintptr_t prev_addr = prev_page->alloc_object_atomic(size);
-        if (prev_addr == 0) {
+        const zaddress prev_addr = prev_page->alloc_object_atomic(size);
+        if (is_null(prev_addr)) {
           // Allocation failed, retry installing the new page
           page = prev_page;
           goto retry;
@@ -134,13 +126,13 @@ uintptr_t ZObjectAllocator::alloc_object_in_shared_page(ZPage** shared_page,
   return addr;
 }
 
-uintptr_t ZObjectAllocator::alloc_large_object(size_t size, ZAllocationFlags flags) {
-  uintptr_t addr = 0;
+zaddress ZObjectAllocator::alloc_large_object(size_t size, ZAllocationFlags flags) {
+  zaddress addr = zaddress::null;
 
   // Allocate new large page
   const size_t page_size = align_up(size, ZGranuleSize);
-  ZPage* const page = alloc_page(ZPageTypeLarge, page_size, flags);
-  if (page != NULL) {
+  ZPage* const page = alloc_page(ZPageType::large, page_size, flags);
+  if (page != nullptr) {
     // Allocate the object
     addr = page->alloc_object(size);
   }
@@ -148,15 +140,15 @@ uintptr_t ZObjectAllocator::alloc_large_object(size_t size, ZAllocationFlags fla
   return addr;
 }
 
-uintptr_t ZObjectAllocator::alloc_medium_object(size_t size, ZAllocationFlags flags) {
-  return alloc_object_in_shared_page(_shared_medium_page.addr(), ZPageTypeMedium, ZPageSizeMedium, size, flags);
+zaddress ZObjectAllocator::alloc_medium_object(size_t size, ZAllocationFlags flags) {
+  return alloc_object_in_shared_page(_shared_medium_page.addr(), ZPageType::medium, ZPageSizeMedium, size, flags);
 }
 
-uintptr_t ZObjectAllocator::alloc_small_object(size_t size, ZAllocationFlags flags) {
-  return alloc_object_in_shared_page(shared_small_page_addr(), ZPageTypeSmall, ZPageSizeSmall, size, flags);
+zaddress ZObjectAllocator::alloc_small_object(size_t size, ZAllocationFlags flags) {
+  return alloc_object_in_shared_page(shared_small_page_addr(), ZPageType::small, ZPageSizeSmall, size, flags);
 }
 
-uintptr_t ZObjectAllocator::alloc_object(size_t size, ZAllocationFlags flags) {
+zaddress ZObjectAllocator::alloc_object(size_t size, ZAllocationFlags flags) {
   if (size <= ZObjectSizeLimitSmall) {
     // Small
     return alloc_small_object(size, flags);
@@ -169,33 +161,26 @@ uintptr_t ZObjectAllocator::alloc_object(size_t size, ZAllocationFlags flags) {
   }
 }
 
-uintptr_t ZObjectAllocator::alloc_object(size_t size) {
-  ZAllocationFlags flags;
+zaddress ZObjectAllocator::alloc_object(size_t size) {
+  const ZAllocationFlags flags;
   return alloc_object(size, flags);
 }
 
-uintptr_t ZObjectAllocator::alloc_object_for_relocation(const ZPageTable* page_table, size_t size) {
+zaddress ZObjectAllocator::alloc_object_for_relocation(size_t size) {
   ZAllocationFlags flags;
   flags.set_non_blocking();
 
-  const uintptr_t addr = alloc_object(size, flags);
-  if (addr != 0) {
-    register_alloc_for_relocation(page_table, addr, size);
-  }
-
-  return addr;
+  return alloc_object(size, flags);
 }
 
-void ZObjectAllocator::undo_alloc_object_for_relocation(ZPage* page, uintptr_t addr, size_t size) {
-  const uint8_t type = page->type();
+void ZObjectAllocator::undo_alloc_object_for_relocation(zaddress addr, size_t size) {
+  ZPage* const page = ZHeap::heap()->page(addr);
 
-  if (type == ZPageTypeLarge) {
-    register_undo_alloc_for_relocation(page, size);
+  if (page->is_large()) {
     undo_alloc_page(page);
     ZStatInc(ZCounterUndoObjectAllocationSucceeded);
   } else {
     if (page->undo_alloc_object_atomic(addr, size)) {
-      register_undo_alloc_for_relocation(page, size);
       ZStatInc(ZCounterUndoObjectAllocationSucceeded);
     } else {
       ZStatInc(ZCounterUndoObjectAllocationFailed);
@@ -203,6 +188,10 @@ void ZObjectAllocator::undo_alloc_object_for_relocation(ZPage* page, uintptr_t a
   }
 }
 
+ZPageAge ZObjectAllocator::age() const {
+  return _age;
+}
+
 size_t ZObjectAllocator::used() const {
   size_t total_used = 0;
   size_t total_undone = 0;
@@ -221,35 +210,16 @@ size_t ZObjectAllocator::used() const {
 }
 
 size_t ZObjectAllocator::remaining() const {
-  assert(ZThread::is_java(), "Should be a Java thread");
+  assert(Thread::current()->is_Java_thread(), "Should be a Java thread");
 
   const ZPage* const page = Atomic::load_acquire(shared_small_page_addr());
-  if (page != NULL) {
+  if (page != nullptr) {
     return page->remaining();
   }
 
   return 0;
 }
 
-size_t ZObjectAllocator::relocated() const {
-  size_t total_alloc = 0;
-  size_t total_undo_alloc = 0;
-
-  ZPerCPUConstIterator<size_t> iter_alloc(&_alloc_for_relocation);
-  for (const size_t* alloc; iter_alloc.next(&alloc);) {
-    total_alloc += Atomic::load(alloc);
-  }
-
-  ZPerCPUConstIterator<size_t> iter_undo_alloc(&_undo_alloc_for_relocation);
-  for (const size_t* undo_alloc; iter_undo_alloc.next(&undo_alloc);) {
-    total_undo_alloc += Atomic::load(undo_alloc);
-  }
-
-  assert(total_alloc >= total_undo_alloc, "Mismatch");
-
-  return total_alloc - total_undo_alloc;
-}
-
 void ZObjectAllocator::retire_pages() {
   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
 
@@ -257,11 +227,7 @@ void ZObjectAllocator::retire_pages() {
   _used.set_all(0);
   _undone.set_all(0);
 
-  // Reset relocated bytes
-  _alloc_for_relocation.set_all(0);
-  _undo_alloc_for_relocation.set_all(0);
-
   // Reset allocation pages
-  _shared_medium_page.set(NULL);
-  _shared_small_page.set_all(NULL);
+  _shared_medium_page.set(nullptr);
+  _shared_small_page.set_all(nullptr);
 }
diff --git a/src/hotspot/share/gc/z/zObjectAllocator.hpp b/src/hotspot/share/gc/z/zObjectAllocator.hpp
index 406782486df..8aa185646fa 100644
--- a/src/hotspot/share/gc/z/zObjectAllocator.hpp
+++ b/src/hotspot/share/gc/z/zObjectAllocator.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,7 +24,10 @@
 #ifndef SHARE_GC_Z_ZOBJECTALLOCATOR_HPP
 #define SHARE_GC_Z_ZOBJECTALLOCATOR_HPP
 
+#include "gc/z/zAddress.hpp"
 #include "gc/z/zAllocationFlags.hpp"
+#include "gc/z/zPageAge.hpp"
+#include "gc/z/zPageType.hpp"
 #include "gc/z/zValue.hpp"
 
 class ZPage;
@@ -32,46 +35,48 @@ class ZPageTable;
 
 class ZObjectAllocator {
 private:
+  ZPageAge           _age;
   const bool         _use_per_cpu_shared_small_pages;
   ZPerCPU<size_t>    _used;
   ZPerCPU<size_t>    _undone;
-  ZPerCPU<size_t>    _alloc_for_relocation;
-  ZPerCPU<size_t>    _undo_alloc_for_relocation;
   ZContended<ZPage*> _shared_medium_page;
   ZPerCPU<ZPage*>    _shared_small_page;
 
   ZPage** shared_small_page_addr();
   ZPage* const* shared_small_page_addr() const;
 
-  void register_alloc_for_relocation(const ZPageTable* page_table, uintptr_t addr, size_t size);
-  void register_undo_alloc_for_relocation(const ZPage* page, size_t size);
-
-  ZPage* alloc_page(uint8_t type, size_t size, ZAllocationFlags flags);
+  ZPage* alloc_page(ZPageType type, size_t size, ZAllocationFlags flags);
   void undo_alloc_page(ZPage* page);
 
   // Allocate an object in a shared page. Allocate and
   // atomically install a new page if necessary.
-  uintptr_t alloc_object_in_shared_page(ZPage** shared_page,
-                                        uint8_t page_type,
-                                        size_t page_size,
-                                        size_t size,
-                                        ZAllocationFlags flags);
+  zaddress alloc_object_in_shared_page(ZPage** shared_page,
+                                       ZPageType page_type,
+                                       size_t page_size,
+                                       size_t size,
+                                       ZAllocationFlags flags);
 
-  uintptr_t alloc_large_object(size_t size, ZAllocationFlags flags);
-  uintptr_t alloc_medium_object(size_t size, ZAllocationFlags flags);
-  uintptr_t alloc_small_object(size_t size, ZAllocationFlags flags);
-  uintptr_t alloc_object(size_t size, ZAllocationFlags flags);
+  zaddress alloc_large_object(size_t size, ZAllocationFlags flags);
+  zaddress alloc_medium_object(size_t size, ZAllocationFlags flags);
+  zaddress alloc_small_object(size_t size, ZAllocationFlags flags);
+  zaddress alloc_object(size_t size, ZAllocationFlags flags);
 
 public:
-  ZObjectAllocator();
+  ZObjectAllocator(ZPageAge age);
 
-  uintptr_t alloc_object(size_t size);
-  uintptr_t alloc_object_for_relocation(const ZPageTable* page_table, size_t size);
-  void undo_alloc_object_for_relocation(ZPage* page, uintptr_t addr, size_t size);
+  // Mutator allocation
+  zaddress alloc_object(size_t size);
+
+  // Relocation
+  zaddress alloc_object_for_relocation(size_t size);
+  void undo_alloc_object_for_relocation(zaddress addr, size_t size);
+
+  ZPage* alloc_page_for_relocation(ZPageType type, size_t size, ZAllocationFlags flags);
+
+  ZPageAge age() const;
 
   size_t used() const;
   size_t remaining() const;
-  size_t relocated() const;
 
   void retire_pages();
 };
diff --git a/src/hotspot/share/gc/z/zPage.cpp b/src/hotspot/share/gc/z/zPage.cpp
index 032834ff13b..52474e95411 100644
--- a/src/hotspot/share/gc/z/zPage.cpp
+++ b/src/hotspot/share/gc/z/zPage.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,79 +22,193 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zList.inline.hpp"
 #include "gc/z/zPage.inline.hpp"
 #include "gc/z/zPhysicalMemory.inline.hpp"
+#include "gc/z/zRememberedSet.inline.hpp"
 #include "gc/z/zVirtualMemory.inline.hpp"
 #include "utilities/align.hpp"
 #include "utilities/debug.hpp"
+#include "utilities/growableArray.hpp"
 
-ZPage::ZPage(const ZVirtualMemory& vmem, const ZPhysicalMemory& pmem) :
-    ZPage(type_from_size(vmem.size()), vmem, pmem) {}
-
-ZPage::ZPage(uint8_t type, const ZVirtualMemory& vmem, const ZPhysicalMemory& pmem) :
+ZPage::ZPage(ZPageType type, const ZVirtualMemory& vmem, const ZPhysicalMemory& pmem) :
     _type(type),
+    _generation_id(ZGenerationId::young),
+    _age(ZPageAge::eden),
     _numa_id((uint8_t)-1),
     _seqnum(0),
+    _seqnum_other(0),
     _virtual(vmem),
-    _top(start()),
+    _top(to_zoffset_end(start())),
     _livemap(object_max_count()),
+    _remembered_set(),
     _last_used(0),
     _physical(pmem),
     _node() {
-  assert_initialized();
-}
-
-ZPage::~ZPage() {}
-
-void ZPage::assert_initialized() const {
   assert(!_virtual.is_null(), "Should not be null");
   assert(!_physical.is_null(), "Should not be null");
   assert(_virtual.size() == _physical.size(), "Virtual/Physical size mismatch");
-  assert((_type == ZPageTypeSmall && size() == ZPageSizeSmall) ||
-         (_type == ZPageTypeMedium && size() == ZPageSizeMedium) ||
-         (_type == ZPageTypeLarge && is_aligned(size(), ZGranuleSize)),
+  assert((_type == ZPageType::small && size() == ZPageSizeSmall) ||
+         (_type == ZPageType::medium && size() == ZPageSizeMedium) ||
+         (_type == ZPageType::large && is_aligned(size(), ZGranuleSize)),
          "Page type/size mismatch");
 }
 
-void ZPage::reset() {
-  _seqnum = ZGlobalSeqNum;
-  _top = start();
-  _livemap.reset();
+ZPage* ZPage::clone_limited() const {
+  // Only copy type and memory layouts. Let the rest be lazily reconstructed when needed.
+  return new ZPage(_type, _virtual, _physical);
+}
+
+ZPage* ZPage::clone_limited_promote_flipped() const {
+  ZPage* const page = new ZPage(_type, _virtual, _physical);
+
+  // The page is still filled with the same objects, need to retain the top pointer.
+  page->_top = _top;
+
+  return page;
+}
+
+ZGeneration* ZPage::generation() {
+  return ZGeneration::generation(_generation_id);
+}
+
+const ZGeneration* ZPage::generation() const {
+  return ZGeneration::generation(_generation_id);
+}
+
+void ZPage::reset_seqnum() {
+  Atomic::store(&_seqnum, generation()->seqnum());
+  Atomic::store(&_seqnum_other, ZGeneration::generation(_generation_id == ZGenerationId::young ? ZGenerationId::old : ZGenerationId::young)->seqnum());
+}
+
+void ZPage::remset_clear() {
+  _remembered_set.clear_all();
+}
+
+void ZPage::verify_remset_after_reset(ZPageAge prev_age, ZPageResetType type) {
+  // Young-to-old reset
+  if (prev_age != ZPageAge::old) {
+    verify_remset_cleared_previous();
+    verify_remset_cleared_current();
+    return;
+  }
+
+  // Old-to-old reset
+  switch (type) {
+  case ZPageResetType::Splitting:
+    // Page is on the way to be destroyed or reused, delay
+    // clearing until the page is reset for Allocation.
+    break;
+
+  case ZPageResetType::InPlaceRelocation:
+    // Relocation failed and page is being compacted in-place.
+    // The remset bits are flipped each young mark start, so
+    // the verification code below needs to use the right remset.
+    if (ZGeneration::old()->active_remset_is_current()) {
+      verify_remset_cleared_previous();
+    } else {
+      verify_remset_cleared_current();
+    }
+    break;
+
+  case ZPageResetType::FlipAging:
+    fatal("Should not have called this for old-to-old flipping");
+    break;
+
+  case ZPageResetType::Allocation:
+    verify_remset_cleared_previous();
+    verify_remset_cleared_current();
+    break;
+  };
+}
+
+void ZPage::reset_remembered_set() {
+  if (is_young()) {
+    // Remset not needed
+    return;
+  }
+
+  // Clearing of remsets is done when freeing a page, so this code only
+  // needs to ensure the remset is initialized the first time a page
+  // becomes old.
+  if (!_remembered_set.is_initialized()) {
+    _remembered_set.initialize(size());
+  }
+}
+
+void ZPage::reset(ZPageAge age, ZPageResetType type) {
+  const ZPageAge prev_age = _age;
+  _age = age;
   _last_used = 0;
+
+  _generation_id = age == ZPageAge::old
+      ? ZGenerationId::old
+      : ZGenerationId::young;
+
+  reset_seqnum();
+
+  // Flip aged pages are still filled with the same objects, need to retain the top pointer.
+  if (type != ZPageResetType::FlipAging) {
+    _top = to_zoffset_end(start());
+  }
+
+  reset_remembered_set();
+  verify_remset_after_reset(prev_age, type);
+
+  if (type != ZPageResetType::InPlaceRelocation || (prev_age != ZPageAge::old && age == ZPageAge::old)) {
+    // Promoted in-place relocations reset the live map,
+    // because they clone the page.
+    _livemap.reset();
+  }
 }
 
-void ZPage::reset_for_in_place_relocation() {
-  _seqnum = ZGlobalSeqNum;
-  _top = start();
+void ZPage::finalize_reset_for_in_place_relocation() {
+  // Now we're done iterating over the livemaps
+  _livemap.reset();
 }
 
-ZPage* ZPage::retype(uint8_t type) {
-  assert(_type != type, "Invalid retype");
+void ZPage::reset_type_and_size(ZPageType type) {
   _type = type;
   _livemap.resize(object_max_count());
+  _remembered_set.resize(size());
+}
+
+ZPage* ZPage::retype(ZPageType type) {
+  assert(_type != type, "Invalid retype");
+  reset_type_and_size(type);
   return this;
 }
 
-ZPage* ZPage::split(size_t size) {
-  return split(type_from_size(size), size);
+ZPage* ZPage::split(size_t split_of_size) {
+  return split(type_from_size(split_of_size), split_of_size);
 }
 
-ZPage* ZPage::split(uint8_t type, size_t size) {
-  assert(_virtual.size() > size, "Invalid split");
+ZPage* ZPage::split_with_pmem(ZPageType type, const ZPhysicalMemory& pmem) {
+  // Resize this page
+  const ZVirtualMemory vmem = _virtual.split(pmem.size());
 
-  // Resize this page, keep _numa_id, _seqnum, and _last_used
-  const ZVirtualMemory vmem = _virtual.split(size);
-  const ZPhysicalMemory pmem = _physical.split(size);
-  _type = type_from_size(_virtual.size());
-  _top = start();
-  _livemap.resize(object_max_count());
+  reset_type_and_size(type_from_size(_virtual.size()));
+  reset(_age, ZPageResetType::Splitting);
 
-  // Create new page, inherit _seqnum and _last_used
-  ZPage* const page = new ZPage(type, vmem, pmem);
-  page->_seqnum = _seqnum;
-  page->_last_used = _last_used;
-  return page;
+  assert(vmem.end() == _virtual.start(), "Should be consecutive");
+
+  log_trace(gc, page)("Split page [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT "]",
+      untype(vmem.start()),
+      untype(vmem.end()),
+      untype(_virtual.end()));
+
+  // Create new page
+  return new ZPage(type, vmem, pmem);
+}
+
+ZPage* ZPage::split(ZPageType type, size_t split_of_size) {
+  assert(_virtual.size() > split_of_size, "Invalid split");
+
+  const ZPhysicalMemory pmem = _physical.split(split_of_size);
+
+  return split_with_pmem(type, pmem);
 }
 
 ZPage* ZPage::split_committed() {
@@ -103,33 +217,101 @@ ZPage* ZPage::split_committed() {
   const ZPhysicalMemory pmem = _physical.split_committed();
   if (pmem.is_null()) {
     // Nothing committed
-    return NULL;
+    return nullptr;
   }
 
   assert(!_physical.is_null(), "Should not be null");
 
-  // Resize this page
-  const ZVirtualMemory vmem = _virtual.split(pmem.size());
-  _type = type_from_size(_virtual.size());
-  _top = start();
-  _livemap.resize(object_max_count());
+  return split_with_pmem(type_from_size(pmem.size()), pmem);
+}
 
-  // Create new page
-  return new ZPage(vmem, pmem);
+class ZFindBaseOopClosure : public ObjectClosure {
+private:
+  volatile zpointer* _p;
+  oop _result;
+
+public:
+  ZFindBaseOopClosure(volatile zpointer* p) :
+      _p(p),
+      _result(nullptr) {}
+
+  virtual void do_object(oop obj) {
+    const uintptr_t p_int = reinterpret_cast<uintptr_t>(_p);
+    const uintptr_t base_int = cast_from_oop<uintptr_t>(obj);
+    const uintptr_t end_int = base_int + wordSize * obj->size();
+    if (p_int >= base_int && p_int < end_int) {
+      _result = obj;
+    }
+  }
+
+  oop result() const { return _result; }
+};
+
+bool ZPage::is_remset_cleared_current() const {
+  return _remembered_set.is_cleared_current();
+}
+
+bool ZPage::is_remset_cleared_previous() const {
+  return _remembered_set.is_cleared_previous();
+}
+
+void ZPage::verify_remset_cleared_current() const {
+  if (ZVerifyRemembered && !is_remset_cleared_current()) {
+    fatal_msg(" current remset bits should be cleared");
+  }
+}
+
+void ZPage::verify_remset_cleared_previous() const {
+  if (ZVerifyRemembered && !is_remset_cleared_previous()) {
+    fatal_msg(" previous remset bits should be cleared");
+  }
+}
+
+void ZPage::clear_remset_current() {
+ _remembered_set.clear_current();
+}
+
+void ZPage::clear_remset_previous() {
+ _remembered_set.clear_previous();
+}
+
+void ZPage::swap_remset_bitmaps() {
+  _remembered_set.swap_remset_bitmaps();
+}
+
+void* ZPage::remset_current() {
+  return _remembered_set.current();
+}
+
+void ZPage::print_on_msg(outputStream* out, const char* msg) const {
+  out->print_cr(" %-6s  " PTR_FORMAT " " PTR_FORMAT " " PTR_FORMAT " %s/%-4u %s%s%s",
+                type_to_string(), untype(start()), untype(top()), untype(end()),
+                is_young() ? "Y" : "O",
+                seqnum(),
+                is_allocating()  ? " Allocating " : "",
+                is_relocatable() ? " Relocatable" : "",
+                msg == nullptr ? "" : msg);
 }
 
 void ZPage::print_on(outputStream* out) const {
-  out->print_cr(" %-6s  " PTR_FORMAT " " PTR_FORMAT " " PTR_FORMAT " %s%s",
-                type_to_string(), start(), top(), end(),
-                is_allocating()  ? " Allocating"  : "",
-                is_relocatable() ? " Relocatable" : "");
+  print_on_msg(out, nullptr);
 }
 
 void ZPage::print() const {
   print_on(tty);
 }
 
-void ZPage::verify_live(uint32_t live_objects, size_t live_bytes) const {
+void ZPage::verify_live(uint32_t live_objects, size_t live_bytes, bool in_place) const {
+  if (!in_place) {
+    // In-place relocation has changed the page to allocating
+    assert_zpage_mark_state();
+  }
   guarantee(live_objects == _livemap.live_objects(), "Invalid number of live objects");
   guarantee(live_bytes == _livemap.live_bytes(), "Invalid number of live bytes");
 }
+
+void ZPage::fatal_msg(const char* msg) const {
+  stringStream ss;
+  print_on_msg(&ss, msg);
+  fatal("%s", ss.base());
+}
diff --git a/src/hotspot/share/gc/z/zPage.hpp b/src/hotspot/share/gc/z/zPage.hpp
index 513773e3f8e..e07b338c710 100644
--- a/src/hotspot/share/gc/z/zPage.hpp
+++ b/src/hotspot/share/gc/z/zPage.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,95 +24,202 @@
 #ifndef SHARE_GC_Z_ZPAGE_HPP
 #define SHARE_GC_Z_ZPAGE_HPP
 
+#include "gc/z/zGenerationId.hpp"
 #include "gc/z/zList.hpp"
 #include "gc/z/zLiveMap.hpp"
+#include "gc/z/zPageAge.hpp"
+#include "gc/z/zPageType.hpp"
 #include "gc/z/zPhysicalMemory.hpp"
+#include "gc/z/zRememberedSet.hpp"
 #include "gc/z/zVirtualMemory.hpp"
 #include "memory/allocation.hpp"
 
+class ZGeneration;
+
+enum class ZPageResetType {
+  // Normal allocation path
+  Allocation,
+  // Relocation failed and started to relocate in-place
+  InPlaceRelocation,
+  // Page was not selected for relocation, all objects
+  // stayed, but the page aged.
+  FlipAging,
+  // The page was split and needs to be reset
+  Splitting,
+};
+
 class ZPage : public CHeapObj<mtGC> {
   friend class VMStructs;
   friend class ZList<ZPage>;
+  friend class ZForwardingTest;
 
 private:
-  uint8_t            _type;
-  uint8_t            _numa_id;
-  uint32_t           _seqnum;
-  ZVirtualMemory     _virtual;
-  volatile uintptr_t _top;
-  ZLiveMap           _livemap;
-  uint64_t           _last_used;
-  ZPhysicalMemory    _physical;
-  ZListNode<ZPage>   _node;
+  ZPageType            _type;
+  ZGenerationId        _generation_id;
+  ZPageAge             _age;
+  uint8_t              _numa_id;
+  uint32_t             _seqnum;
+  uint32_t             _seqnum_other;
+  ZVirtualMemory       _virtual;
+  volatile zoffset_end _top;
+  ZLiveMap             _livemap;
+  ZRememberedSet       _remembered_set;
+  uint64_t             _last_used;
+  ZPhysicalMemory      _physical;
+  ZListNode<ZPage>     _node;
 
-  void assert_initialized() const;
-
-  uint8_t type_from_size(size_t size) const;
+  ZPageType type_from_size(size_t size) const;
   const char* type_to_string() const;
 
-  bool is_object_marked(uintptr_t addr) const;
-  bool is_object_strongly_marked(uintptr_t addr) const;
+  BitMap::idx_t bit_index(zaddress addr) const;
+  zoffset offset_from_bit_index(BitMap::idx_t index) const;
+  oop object_from_bit_index(BitMap::idx_t index) const;
+
+  bool is_live_bit_set(zaddress addr) const;
+  bool is_strong_bit_set(zaddress addr) const;
+
+  ZGeneration* generation();
+  const ZGeneration* generation() const;
+
+  void reset_seqnum();
+  void reset_remembered_set();
+
+  ZPage* split_with_pmem(ZPageType type, const ZPhysicalMemory& pmem);
+
+  void verify_remset_after_reset(ZPageAge prev_age, ZPageResetType type);
 
 public:
-  ZPage(const ZVirtualMemory& vmem, const ZPhysicalMemory& pmem);
-  ZPage(uint8_t type, const ZVirtualMemory& vmem, const ZPhysicalMemory& pmem);
-  ~ZPage();
+  ZPage(ZPageType type, const ZVirtualMemory& vmem, const ZPhysicalMemory& pmem);
+
+  ZPage* clone_limited() const;
+  ZPage* clone_limited_promote_flipped() const;
 
   uint32_t object_max_count() const;
   size_t object_alignment_shift() const;
   size_t object_alignment() const;
 
-  uint8_t type() const;
-  uintptr_t start() const;
-  uintptr_t end() const;
+  ZPageType type() const;
+
+  bool is_small() const;
+  bool is_medium() const;
+  bool is_large() const;
+
+  ZGenerationId generation_id() const;
+  bool is_young() const;
+  bool is_old() const;
+  zoffset start() const;
+  zoffset_end end() const;
   size_t size() const;
-  uintptr_t top() const;
+  zoffset_end top() const;
   size_t remaining() const;
+  size_t used() const;
 
   const ZVirtualMemory& virtual_memory() const;
   const ZPhysicalMemory& physical_memory() const;
   ZPhysicalMemory& physical_memory();
 
   uint8_t numa_id();
+  ZPageAge age() const;
 
+  uint32_t seqnum() const;
   bool is_allocating() const;
   bool is_relocatable() const;
 
   uint64_t last_used() const;
   void set_last_used();
 
-  void reset();
-  void reset_for_in_place_relocation();
+  void reset(ZPageAge age, ZPageResetType type);
 
-  ZPage* retype(uint8_t type);
-  ZPage* split(size_t size);
-  ZPage* split(uint8_t type, size_t size);
+  void finalize_reset_for_in_place_relocation();
+
+  void reset_type_and_size(ZPageType type);
+
+  ZPage* retype(ZPageType type);
+  ZPage* split(size_t split_of_size);
+  ZPage* split(ZPageType type, size_t split_of_size);
   ZPage* split_committed();
 
-  bool is_in(uintptr_t addr) const;
+  bool is_in(zoffset offset) const;
+  bool is_in(zaddress addr) const;
+
+  uintptr_t local_offset(zoffset offset) const;
+  uintptr_t local_offset(zoffset_end offset) const;
+  uintptr_t local_offset(zaddress addr) const;
+  uintptr_t local_offset(zaddress_unsafe addr) const;
+
+  zoffset global_offset(uintptr_t local_offset) const;
+
+  bool is_object_live(zaddress addr) const;
+  bool is_object_strongly_live(zaddress addr) const;
 
   bool is_marked() const;
-  template <bool finalizable> bool is_object_marked(uintptr_t addr) const;
-  bool is_object_live(uintptr_t addr) const;
-  bool is_object_strongly_live(uintptr_t addr) const;
-  bool mark_object(uintptr_t addr, bool finalizable, bool& inc_live);
+  bool is_object_marked_live(zaddress addr) const;
+  bool is_object_marked_strong(zaddress addr) const;
+  bool is_object_marked(zaddress addr, bool finalizable) const;
+  bool mark_object(zaddress addr, bool finalizable, bool& inc_live);
 
   void inc_live(uint32_t objects, size_t bytes);
   uint32_t live_objects() const;
   size_t live_bytes() const;
 
-  void object_iterate(ObjectClosure* cl);
+  template <typename Function>
+  void object_iterate(Function function);
 
-  uintptr_t alloc_object(size_t size);
-  uintptr_t alloc_object_atomic(size_t size);
+  void remember(volatile zpointer* p);
 
-  bool undo_alloc_object(uintptr_t addr, size_t size);
-  bool undo_alloc_object_atomic(uintptr_t addr, size_t size);
+  // In-place relocation support
+  void clear_remset_bit_non_par_current(uintptr_t l_offset);
+  void clear_remset_range_non_par_current(uintptr_t l_offset, size_t size);
+  void swap_remset_bitmaps();
 
+  void remset_clear();
+
+  ZBitMap::ReverseIterator remset_reverse_iterator_previous();
+  BitMap::Iterator remset_iterator_limited_current(uintptr_t l_offset, size_t size);
+  BitMap::Iterator remset_iterator_limited_previous(uintptr_t l_offset, size_t size);
+
+  zaddress_unsafe find_base_unsafe(volatile zpointer* p);
+  zaddress_unsafe find_base(volatile zpointer* p);
+
+  template <typename Function>
+  void oops_do_remembered(Function function);
+
+  // Only visits remembered set entries for live objects
+  template <typename Function>
+  void oops_do_remembered_in_live(Function function);
+
+  template <typename Function>
+  void oops_do_current_remembered(Function function);
+
+  bool is_remset_cleared_current() const;
+  bool is_remset_cleared_previous() const;
+
+  void verify_remset_cleared_current() const;
+  void verify_remset_cleared_previous() const;
+
+  void clear_remset_current();
+  void clear_remset_previous();
+
+  void* remset_current();
+
+  zaddress alloc_object(size_t size);
+  zaddress alloc_object_atomic(size_t size);
+
+  bool undo_alloc_object(zaddress addr, size_t size);
+  bool undo_alloc_object_atomic(zaddress addr, size_t size);
+
+  void log_msg(const char* msg_format, ...) const ATTRIBUTE_PRINTF(2, 3);
+
+  void print_on_msg(outputStream* out, const char* msg) const;
   void print_on(outputStream* out) const;
   void print() const;
 
-  void verify_live(uint32_t live_objects, size_t live_bytes) const;
+  // Verification
+  bool was_remembered(volatile zpointer* p);
+  bool is_remembered(volatile zpointer* p);
+  void verify_live(uint32_t live_objects, size_t live_bytes, bool in_place) const;
+
+  void fatal_msg(const char* msg) const;
 };
 
 class ZPageClosure {
diff --git a/src/hotspot/share/gc/z/zPage.inline.hpp b/src/hotspot/share/gc/z/zPage.inline.hpp
index e7d6742b91c..6cacf699145 100644
--- a/src/hotspot/share/gc/z/zPage.inline.hpp
+++ b/src/hotspot/share/gc/z/zPage.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,43 +27,50 @@
 #include "gc/z/zPage.hpp"
 
 #include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zLiveMap.inline.hpp"
 #include "gc/z/zNUMA.hpp"
 #include "gc/z/zPhysicalMemory.inline.hpp"
+#include "gc/z/zRememberedSet.inline.hpp"
+#include "gc/z/zUtils.inline.hpp"
 #include "gc/z/zVirtualMemory.inline.hpp"
+#include "logging/logStream.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/os.hpp"
 #include "utilities/align.hpp"
 #include "utilities/debug.hpp"
 
-inline uint8_t ZPage::type_from_size(size_t size) const {
+inline ZPageType ZPage::type_from_size(size_t size) const {
   if (size == ZPageSizeSmall) {
-    return ZPageTypeSmall;
+    return ZPageType::small;
   } else if (size == ZPageSizeMedium) {
-    return ZPageTypeMedium;
+    return ZPageType::medium;
   } else {
-    return ZPageTypeLarge;
+    return ZPageType::large;
   }
 }
 
 inline const char* ZPage::type_to_string() const {
   switch (type()) {
-  case ZPageTypeSmall:
+  case ZPageType::small:
     return "Small";
 
-  case ZPageTypeMedium:
+  case ZPageType::medium:
     return "Medium";
 
-  default:
-    assert(type() == ZPageTypeLarge, "Invalid page type");
+  case ZPageType::large:
     return "Large";
+
+  default:
+    fatal("Unexpected page type");
+    return 0;
   }
 }
 
 inline uint32_t ZPage::object_max_count() const {
   switch (type()) {
-  case ZPageTypeLarge:
+  case ZPageType::large:
     // A large page can only contain a single
     // object aligned to the start of the page.
     return 1;
@@ -75,41 +82,71 @@ inline uint32_t ZPage::object_max_count() const {
 
 inline size_t ZPage::object_alignment_shift() const {
   switch (type()) {
-  case ZPageTypeSmall:
+  case ZPageType::small:
     return ZObjectAlignmentSmallShift;
 
-  case ZPageTypeMedium:
+  case ZPageType::medium:
     return ZObjectAlignmentMediumShift;
 
-  default:
-    assert(type() == ZPageTypeLarge, "Invalid page type");
+  case ZPageType::large:
     return ZObjectAlignmentLargeShift;
+
+  default:
+    fatal("Unexpected page type");
+    return 0;
   }
 }
 
 inline size_t ZPage::object_alignment() const {
   switch (type()) {
-  case ZPageTypeSmall:
+  case ZPageType::small:
     return ZObjectAlignmentSmall;
 
-  case ZPageTypeMedium:
+  case ZPageType::medium:
     return ZObjectAlignmentMedium;
 
-  default:
-    assert(type() == ZPageTypeLarge, "Invalid page type");
+  case ZPageType::large:
     return ZObjectAlignmentLarge;
+
+  default:
+    fatal("Unexpected page type");
+    return 0;
   }
 }
 
-inline uint8_t ZPage::type() const {
+inline ZPageType ZPage::type() const {
   return _type;
 }
 
-inline uintptr_t ZPage::start() const {
+inline bool ZPage::is_small() const {
+  return _type == ZPageType::small;
+}
+
+inline bool ZPage::is_medium() const {
+  return _type == ZPageType::medium;
+}
+
+inline bool ZPage::is_large() const {
+  return _type == ZPageType::large;
+}
+
+inline ZGenerationId ZPage::generation_id() const {
+  return _generation_id;
+}
+
+inline bool ZPage::is_young() const {
+  return _generation_id == ZGenerationId::young;
+}
+
+inline bool ZPage::is_old() const {
+  return _generation_id == ZGenerationId::old;
+}
+
+inline zoffset ZPage::start() const {
   return _virtual.start();
 }
 
-inline uintptr_t ZPage::end() const {
+inline zoffset_end ZPage::end() const {
   return _virtual.end();
 }
 
@@ -117,7 +154,7 @@ inline size_t ZPage::size() const {
   return _virtual.size();
 }
 
-inline uintptr_t ZPage::top() const {
+inline zoffset_end ZPage::top() const {
   return _top;
 }
 
@@ -125,6 +162,10 @@ inline size_t ZPage::remaining() const {
   return end() - top();
 }
 
+inline size_t ZPage::used() const {
+  return top() - start();
+}
+
 inline const ZVirtualMemory& ZPage::virtual_memory() const {
   return _virtual;
 }
@@ -139,18 +180,26 @@ inline ZPhysicalMemory& ZPage::physical_memory() {
 
 inline uint8_t ZPage::numa_id() {
   if (_numa_id == (uint8_t)-1) {
-    _numa_id = ZNUMA::memory_id(ZAddress::good(start()));
+    _numa_id = ZNUMA::memory_id(untype(ZOffset::address(start())));
   }
 
   return _numa_id;
 }
 
+inline ZPageAge ZPage::age() const {
+  return _age;
+}
+
+inline uint32_t ZPage::seqnum() const {
+  return _seqnum;
+}
+
 inline bool ZPage::is_allocating() const {
-  return _seqnum == ZGlobalSeqNum;
+  return _seqnum == generation()->seqnum();
 }
 
 inline bool ZPage::is_relocatable() const {
-  return _seqnum < ZGlobalSeqNum;
+  return _seqnum < generation()->seqnum();
 }
 
 inline uint64_t ZPage::last_used() const {
@@ -161,103 +210,293 @@ inline void ZPage::set_last_used() {
   _last_used = ceil(os::elapsedTime());
 }
 
-inline bool ZPage::is_in(uintptr_t addr) const {
-  const uintptr_t offset = ZAddress::offset(addr);
+inline bool ZPage::is_in(zoffset offset) const {
   return offset >= start() && offset < top();
 }
 
+inline bool ZPage::is_in(zaddress addr) const {
+  const zoffset offset = ZAddress::offset(addr);
+  return is_in(offset);
+}
+
+inline uintptr_t ZPage::local_offset(zoffset offset) const {
+  assert(ZHeap::heap()->is_in_page_relaxed(this, ZOffset::address(offset)),
+         "Invalid offset " PTR_FORMAT " page [" PTR_FORMAT ", " PTR_FORMAT ", " PTR_FORMAT ")",
+         untype(offset), untype(start()), untype(top()), untype(end()));
+  return offset - start();
+}
+
+inline uintptr_t ZPage::local_offset(zoffset_end offset) const {
+  assert(offset <= end(), "Wrong offset");
+  return offset - start();
+}
+
+inline uintptr_t ZPage::local_offset(zaddress addr) const {
+  const zoffset offset = ZAddress::offset(addr);
+  return local_offset(offset);
+}
+
+inline uintptr_t ZPage::local_offset(zaddress_unsafe addr) const {
+  const zoffset offset = ZAddress::offset(addr);
+  return local_offset(offset);
+}
+
+inline zoffset ZPage::global_offset(uintptr_t local_offset) const {
+  return start() + local_offset;
+}
+
 inline bool ZPage::is_marked() const {
   assert(is_relocatable(), "Invalid page state");
-  return _livemap.is_marked();
+  return _livemap.is_marked(_generation_id);
 }
 
-inline bool ZPage::is_object_marked(uintptr_t addr) const {
+inline BitMap::idx_t ZPage::bit_index(zaddress addr) const {
+  return (local_offset(addr) >> object_alignment_shift()) * 2;
+}
+
+inline zoffset ZPage::offset_from_bit_index(BitMap::idx_t index) const {
+  const uintptr_t l_offset = ((index / 2) << object_alignment_shift());
+  return start() + l_offset;
+}
+
+inline oop ZPage::object_from_bit_index(BitMap::idx_t index) const {
+  const zoffset offset = offset_from_bit_index(index);
+  return to_oop(ZOffset::address(offset));
+}
+
+inline bool ZPage::is_live_bit_set(zaddress addr) const {
   assert(is_relocatable(), "Invalid page state");
-  const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
-  return _livemap.get(index);
+  const BitMap::idx_t index = bit_index(addr);
+  return _livemap.get(_generation_id, index);
 }
 
-inline bool ZPage::is_object_strongly_marked(uintptr_t addr) const {
+inline bool ZPage::is_strong_bit_set(zaddress addr) const {
   assert(is_relocatable(), "Invalid page state");
-  const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
-  return _livemap.get(index + 1);
+  const BitMap::idx_t index = bit_index(addr);
+  return _livemap.get(_generation_id, index + 1);
 }
 
-template <bool finalizable>
-inline bool ZPage::is_object_marked(uintptr_t addr) const {
-  return finalizable ? is_object_marked(addr) : is_object_strongly_marked(addr);
+inline bool ZPage::is_object_live(zaddress addr) const {
+  return is_allocating() || is_live_bit_set(addr);
 }
 
-inline bool ZPage::is_object_live(uintptr_t addr) const {
-  return is_allocating() || is_object_marked(addr);
+inline bool ZPage::is_object_strongly_live(zaddress addr) const {
+  return is_allocating() || is_strong_bit_set(addr);
 }
 
-inline bool ZPage::is_object_strongly_live(uintptr_t addr) const {
-  return is_allocating() || is_object_strongly_marked(addr);
+inline bool ZPage::is_object_marked_live(zaddress addr) const {
+  // This function is only used by the marking code and therefore has stronger
+  // asserts that are not always valid to ask when checking for liveness.
+  assert(!is_old() || ZGeneration::old()->is_phase_mark(), "Location should match phase");
+  assert(!is_young() || ZGeneration::young()->is_phase_mark(), "Location should match phase");
+
+  return is_object_live(addr);
 }
 
-inline bool ZPage::mark_object(uintptr_t addr, bool finalizable, bool& inc_live) {
-  assert(ZAddress::is_marked(addr), "Invalid address");
+inline bool ZPage::is_object_marked_strong(zaddress addr) const {
+  // This function is only used by the marking code and therefore has stronger
+  // asserts that are not always valid to ask when checking for liveness.
+  assert(!is_old() || ZGeneration::old()->is_phase_mark(), "Location should match phase");
+  assert(!is_young() || ZGeneration::young()->is_phase_mark(), "Location should match phase");
+
+  return is_object_strongly_live(addr);
+}
+
+inline bool ZPage::is_object_marked(zaddress addr, bool finalizable) const {
+  return finalizable ? is_object_marked_live(addr) : is_object_marked_strong(addr);
+}
+
+inline bool ZPage::mark_object(zaddress addr, bool finalizable, bool& inc_live) {
   assert(is_relocatable(), "Invalid page state");
   assert(is_in(addr), "Invalid address");
 
+  // Verify oop
+  (void)to_oop(addr);
+
   // Set mark bit
-  const size_t index = ((ZAddress::offset(addr) - start()) >> object_alignment_shift()) * 2;
-  return _livemap.set(index, finalizable, inc_live);
+  const BitMap::idx_t index = bit_index(addr);
+  return _livemap.set(_generation_id, index, finalizable, inc_live);
 }
 
 inline void ZPage::inc_live(uint32_t objects, size_t bytes) {
   _livemap.inc_live(objects, bytes);
 }
 
+#define assert_zpage_mark_state()                                                  \
+  do {                                                                             \
+    assert(is_marked(), "Should be marked");                                       \
+    assert(!is_young() || !ZGeneration::young()->is_phase_mark(), "Wrong phase");  \
+    assert(!is_old() || !ZGeneration::old()->is_phase_mark(), "Wrong phase");      \
+  } while (0)
+
 inline uint32_t ZPage::live_objects() const {
-  assert(is_marked(), "Should be marked");
+  assert_zpage_mark_state();
+
   return _livemap.live_objects();
 }
 
 inline size_t ZPage::live_bytes() const {
-  assert(is_marked(), "Should be marked");
+  assert_zpage_mark_state();
+
   return _livemap.live_bytes();
 }
 
-inline void ZPage::object_iterate(ObjectClosure* cl) {
-  _livemap.iterate(cl, ZAddress::good(start()), object_alignment_shift());
+template <typename Function>
+inline void ZPage::object_iterate(Function function) {
+  auto do_bit = [&](BitMap::idx_t index) -> bool {
+    const oop obj = object_from_bit_index(index);
+
+    // Apply function
+    function(obj);
+
+    return true;
+  };
+
+  _livemap.iterate(_generation_id, do_bit);
 }
 
-inline uintptr_t ZPage::alloc_object(size_t size) {
+inline void ZPage::remember(volatile zpointer* p) {
+  const zaddress addr = to_zaddress((uintptr_t)p);
+  const uintptr_t l_offset = local_offset(addr);
+  _remembered_set.set_current(l_offset);
+}
+
+inline void ZPage::clear_remset_bit_non_par_current(uintptr_t l_offset) {
+  _remembered_set.unset_non_par_current(l_offset);
+}
+
+inline void ZPage::clear_remset_range_non_par_current(uintptr_t l_offset, size_t size) {
+  _remembered_set.unset_range_non_par_current(l_offset, size);
+}
+
+inline ZBitMap::ReverseIterator ZPage::remset_reverse_iterator_previous() {
+  return _remembered_set.iterator_reverse_previous();
+}
+
+inline BitMap::Iterator ZPage::remset_iterator_limited_current(uintptr_t l_offset, size_t size) {
+  return _remembered_set.iterator_limited_current(l_offset, size);
+}
+
+inline BitMap::Iterator ZPage::remset_iterator_limited_previous(uintptr_t l_offset, size_t size) {
+  return _remembered_set.iterator_limited_previous(l_offset, size);
+}
+
+inline bool ZPage::is_remembered(volatile zpointer* p) {
+  const zaddress addr = to_zaddress((uintptr_t)p);
+  const uintptr_t l_offset = local_offset(addr);
+  return _remembered_set.at_current(l_offset);
+}
+
+inline bool ZPage::was_remembered(volatile zpointer* p) {
+  const zaddress addr = to_zaddress((uintptr_t)p);
+  const uintptr_t l_offset = local_offset(addr);
+  return _remembered_set.at_previous(l_offset);
+}
+
+
+inline zaddress_unsafe ZPage::find_base_unsafe(volatile zpointer* p) {
+  if (is_large()) {
+    return ZOffset::address_unsafe(start());
+  }
+
+  // Note: when thinking about excluding looking at the index corresponding to
+  // the field address p, it's important to note that for medium pages both p
+  // and it's associated base could map to the same index.
+  const BitMap::idx_t index = bit_index(zaddress(uintptr_t(p)));
+  const BitMap::idx_t base_index = _livemap.find_base_bit(index);
+  if (base_index == BitMap::idx_t(-1)) {
+    return zaddress_unsafe::null;
+  } else {
+    return ZOffset::address_unsafe(offset_from_bit_index(base_index));
+  }
+}
+
+inline zaddress_unsafe ZPage::find_base(volatile zpointer* p) {
+  assert_zpage_mark_state();
+
+  return find_base_unsafe(p);
+}
+
+template <typename Function>
+inline void ZPage::oops_do_remembered(Function function) {
+  _remembered_set.iterate_previous([&](uintptr_t local_offset) {
+    const zoffset offset = start() + local_offset;
+    const zaddress addr = ZOffset::address(offset);
+
+    function((volatile zpointer*)addr);
+  });
+}
+
+template <typename Function>
+inline void ZPage::oops_do_remembered_in_live(Function function) {
+  assert(!is_allocating(), "Must have liveness information");
+  assert(!ZGeneration::old()->is_phase_mark(), "Must have liveness information");
+  assert(is_marked(), "Must have liveness information");
+
+  ZRememberedSetContainingInLiveIterator iter(this);
+  for (ZRememberedSetContaining containing; iter.next(&containing);) {
+    function((volatile zpointer*)containing._field_addr);
+  }
+
+  iter.print_statistics();
+}
+
+template <typename Function>
+inline void ZPage::oops_do_current_remembered(Function function) {
+  _remembered_set.iterate_current([&](uintptr_t local_offset) {
+    const zoffset offset = start() + local_offset;
+    const zaddress addr = ZOffset::address(offset);
+
+    function((volatile zpointer*)addr);
+  });
+}
+
+inline zaddress ZPage::alloc_object(size_t size) {
   assert(is_allocating(), "Invalid state");
 
   const size_t aligned_size = align_up(size, object_alignment());
-  const uintptr_t addr = top();
-  const uintptr_t new_top = addr + aligned_size;
+  const zoffset_end addr = top();
+
+  zoffset_end new_top;
+
+  if (!to_zoffset_end(&new_top, addr, aligned_size)) {
+    // Next top would be outside of the heap - bail
+    return zaddress::null;
+  }
 
   if (new_top > end()) {
-    // Not enough space left
-    return 0;
+    // Not enough space left in the page
+    return zaddress::null;
   }
 
   _top = new_top;
 
-  return ZAddress::good(addr);
+  return ZOffset::address(to_zoffset(addr));
 }
 
-inline uintptr_t ZPage::alloc_object_atomic(size_t size) {
+inline zaddress ZPage::alloc_object_atomic(size_t size) {
   assert(is_allocating(), "Invalid state");
 
   const size_t aligned_size = align_up(size, object_alignment());
-  uintptr_t addr = top();
+  zoffset_end addr = top();
 
   for (;;) {
-    const uintptr_t new_top = addr + aligned_size;
-    if (new_top > end()) {
-      // Not enough space left
-      return 0;
+    zoffset_end new_top;
+
+    if (!to_zoffset_end(&new_top, addr, aligned_size)) {
+      // Next top would be outside of the heap - bail
+      return zaddress::null;
     }
 
-    const uintptr_t prev_top = Atomic::cmpxchg(&_top, addr, new_top);
+    if (new_top > end()) {
+      // Not enough space left
+      return zaddress::null;
+    }
+
+    const zoffset_end prev_top = Atomic::cmpxchg(&_top, addr, new_top);
     if (prev_top == addr) {
       // Success
-      return ZAddress::good(addr);
+      return ZOffset::address(to_zoffset(addr));
     }
 
     // Retry
@@ -265,13 +504,13 @@ inline uintptr_t ZPage::alloc_object_atomic(size_t size) {
   }
 }
 
-inline bool ZPage::undo_alloc_object(uintptr_t addr, size_t size) {
+inline bool ZPage::undo_alloc_object(zaddress addr, size_t size) {
   assert(is_allocating(), "Invalid state");
 
-  const uintptr_t offset = ZAddress::offset(addr);
+  const zoffset offset = ZAddress::offset(addr);
   const size_t aligned_size = align_up(size, object_alignment());
-  const uintptr_t old_top = top();
-  const uintptr_t new_top = old_top - aligned_size;
+  const zoffset_end old_top = top();
+  const zoffset_end new_top = old_top - aligned_size;
 
   if (new_top != offset) {
     // Failed to undo allocation, not the last allocated object
@@ -284,21 +523,21 @@ inline bool ZPage::undo_alloc_object(uintptr_t addr, size_t size) {
   return true;
 }
 
-inline bool ZPage::undo_alloc_object_atomic(uintptr_t addr, size_t size) {
+inline bool ZPage::undo_alloc_object_atomic(zaddress addr, size_t size) {
   assert(is_allocating(), "Invalid state");
 
-  const uintptr_t offset = ZAddress::offset(addr);
+  const zoffset offset = ZAddress::offset(addr);
   const size_t aligned_size = align_up(size, object_alignment());
-  uintptr_t old_top = top();
+  zoffset_end old_top = top();
 
   for (;;) {
-    const uintptr_t new_top = old_top - aligned_size;
+    const zoffset_end new_top = old_top - aligned_size;
     if (new_top != offset) {
       // Failed to undo allocation, not the last allocated object
       return false;
     }
 
-    const uintptr_t prev_top = Atomic::cmpxchg(&_top, old_top, new_top);
+    const zoffset_end prev_top = Atomic::cmpxchg(&_top, old_top, new_top);
     if (prev_top == old_top) {
       // Success
       return true;
@@ -309,4 +548,15 @@ inline bool ZPage::undo_alloc_object_atomic(uintptr_t addr, size_t size) {
   }
 }
 
+inline void ZPage::log_msg(const char* msg_format, ...) const {
+  LogTarget(Trace, gc, page) target;
+  if (target.is_enabled()) {
+    va_list argp;
+    va_start(argp, msg_format);
+    LogStream stream(target);
+    print_on_msg(&stream, err_msg(FormatBufferDummy(), msg_format, argp));
+    va_end(argp);
+  }
+}
+
 #endif // SHARE_GC_Z_ZPAGE_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zPageAge.hpp b/src/hotspot/share/gc/z/zPageAge.hpp
new file mode 100644
index 00000000000..b7b1688b830
--- /dev/null
+++ b/src/hotspot/share/gc/z/zPageAge.hpp
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZPAGEAGE_HPP
+#define SHARE_GC_Z_ZPAGEAGE_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+enum class ZPageAge : uint8_t {
+  eden,
+  survivor1,
+  survivor2,
+  survivor3,
+  survivor4,
+  survivor5,
+  survivor6,
+  survivor7,
+  survivor8,
+  survivor9,
+  survivor10,
+  survivor11,
+  survivor12,
+  survivor13,
+  survivor14,
+  old
+};
+
+constexpr uint ZPageAgeMax = static_cast<uint>(ZPageAge::old);
+
+#endif // SHARE_GC_Z_ZPAGEAGE_HPP
diff --git a/src/hotspot/share/gc/z/zPageAllocator.cpp b/src/hotspot/share/gc/z/zPageAllocator.cpp
index e964aec5454..18349483623 100644
--- a/src/hotspot/share/gc/z/zPageAllocator.cpp
+++ b/src/hotspot/share/gc/z/zPageAllocator.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,11 +25,14 @@
 #include "gc/shared/gcLogPrecious.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "gc/z/zArray.inline.hpp"
-#include "gc/z/zCollectedHeap.hpp"
+#include "gc/z/zDriver.hpp"
 #include "gc/z/zFuture.inline.hpp"
+#include "gc/z/zGeneration.inline.hpp"
+#include "gc/z/zGenerationId.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zLock.inline.hpp"
 #include "gc/z/zPage.inline.hpp"
+#include "gc/z/zPageAge.hpp"
 #include "gc/z/zPageAllocator.inline.hpp"
 #include "gc/z/zPageCache.hpp"
 #include "gc/z/zSafeDelete.inline.hpp"
@@ -46,44 +49,82 @@
 #include "utilities/debug.hpp"
 #include "utilities/globalDefinitions.hpp"
 
-static const ZStatCounter       ZCounterAllocationRate("Memory", "Allocation Rate", ZStatUnitBytesPerSecond);
+static const ZStatCounter       ZCounterMutatorAllocationRate("Memory", "Allocation Rate", ZStatUnitBytesPerSecond);
 static const ZStatCounter       ZCounterPageCacheFlush("Memory", "Page Cache Flush", ZStatUnitBytesPerSecond);
 static const ZStatCounter       ZCounterDefragment("Memory", "Defragment", ZStatUnitOpsPerSecond);
 static const ZStatCriticalPhase ZCriticalPhaseAllocationStall("Allocation Stall");
 
-enum ZPageAllocationStall {
-  ZPageAllocationStallSuccess,
-  ZPageAllocationStallFailed,
-  ZPageAllocationStallStartGC
-};
+ZSafePageRecycle::ZSafePageRecycle(ZPageAllocator* page_allocator) :
+    _page_allocator(page_allocator),
+    _unsafe_to_recycle() {}
+
+void ZSafePageRecycle::activate() {
+  _unsafe_to_recycle.activate();
+}
+
+void ZSafePageRecycle::deactivate() {
+  auto delete_function = [&](ZPage* page) {
+    _page_allocator->safe_destroy_page(page);
+  };
+
+  _unsafe_to_recycle.deactivate_and_apply(delete_function);
+}
+
+ZPage* ZSafePageRecycle::register_and_clone_if_activated(ZPage* page) {
+  if (!_unsafe_to_recycle.is_activated()) {
+    // The page has no concurrent readers.
+    // Recycle original page.
+    return page;
+  }
+
+  // The page could have concurrent readers.
+  // It would be unsafe to recycle this page at this point.
+
+  // As soon as the page is added to _unsafe_to_recycle, it
+  // must not be used again. Hence, the extra double-checked
+  // locking to only clone the page if it is believed to be
+  // unsafe to recycle the page.
+  ZPage* const cloned_page = page->clone_limited();
+  if (!_unsafe_to_recycle.add_if_activated(page)) {
+    // It became safe to recycle the page after the is_activated check
+    delete cloned_page;
+    return page;
+  }
+
+  // The original page has been registered to be deleted by another thread.
+  // Recycle the cloned page.
+  return cloned_page;
+}
 
 class ZPageAllocation : public StackObj {
   friend class ZList<ZPageAllocation>;
 
 private:
-  const uint8_t                 _type;
-  const size_t                  _size;
-  const ZAllocationFlags        _flags;
-  const uint32_t                _seqnum;
-  size_t                        _flushed;
-  size_t                        _committed;
-  ZList<ZPage>                  _pages;
-  ZListNode<ZPageAllocation>    _node;
-  ZFuture<ZPageAllocationStall> _stall_result;
+  const ZPageType            _type;
+  const size_t               _size;
+  const ZAllocationFlags     _flags;
+  const uint32_t             _young_seqnum;
+  const uint32_t             _old_seqnum;
+  size_t                     _flushed;
+  size_t                     _committed;
+  ZList<ZPage>               _pages;
+  ZListNode<ZPageAllocation> _node;
+  ZFuture<bool>              _stall_result;
 
 public:
-  ZPageAllocation(uint8_t type, size_t size, ZAllocationFlags flags) :
+  ZPageAllocation(ZPageType type, size_t size, ZAllocationFlags flags) :
       _type(type),
       _size(size),
       _flags(flags),
-      _seqnum(ZGlobalSeqNum),
+      _young_seqnum(ZGeneration::young()->seqnum()),
+      _old_seqnum(ZGeneration::old()->seqnum()),
       _flushed(0),
       _committed(0),
       _pages(),
       _node(),
       _stall_result() {}
 
-  uint8_t type() const {
+  ZPageType type() const {
     return _type;
   }
 
@@ -95,8 +136,12 @@ public:
     return _flags;
   }
 
-  uint32_t seqnum() const {
-    return _seqnum;
+  uint32_t young_seqnum() const {
+    return _young_seqnum;
+  }
+
+  uint32_t old_seqnum() const {
+    return _old_seqnum;
   }
 
   size_t flushed() const {
@@ -115,7 +160,7 @@ public:
     _committed = committed;
   }
 
-  ZPageAllocationStall wait() {
+  bool wait() {
     return _stall_result.get();
   }
 
@@ -123,34 +168,37 @@ public:
     return &_pages;
   }
 
-  void satisfy(ZPageAllocationStall result) {
+  void satisfy(bool result) {
     _stall_result.set(result);
   }
+
+  bool gc_relocation() const {
+    return _flags.gc_relocation();
+  }
 };
 
-ZPageAllocator::ZPageAllocator(ZWorkers* workers,
-                               size_t min_capacity,
+ZPageAllocator::ZPageAllocator(size_t min_capacity,
                                size_t initial_capacity,
+                               size_t soft_max_capacity,
                                size_t max_capacity) :
     _lock(),
     _cache(),
     _virtual(max_capacity),
     _physical(max_capacity),
     _min_capacity(min_capacity),
+    _initial_capacity(initial_capacity),
     _max_capacity(max_capacity),
     _current_max_capacity(max_capacity),
     _capacity(0),
     _claimed(0),
     _used(0),
-    _used_high(0),
-    _used_low(0),
-    _reclaimed(0),
+    _used_generations{0, 0},
+    _collection_stats{{0, 0}, {0, 0}},
     _stalled(),
-    _nstalled(0),
-    _satisfied(),
     _unmapper(new ZUnmapper(this)),
     _uncommitter(new ZUncommitter(this)),
-    _safe_delete(),
+    _safe_destroy(),
+    _safe_recycle(this),
     _initialized(false) {
 
   if (!_virtual.is_initialized() || !_physical.is_initialized()) {
@@ -160,6 +208,7 @@ ZPageAllocator::ZPageAllocator(ZWorkers* workers,
   log_info_p(gc, init)("Min Capacity: " SIZE_FORMAT "M", min_capacity / M);
   log_info_p(gc, init)("Initial Capacity: " SIZE_FORMAT "M", initial_capacity / M);
   log_info_p(gc, init)("Max Capacity: " SIZE_FORMAT "M", max_capacity / M);
+  log_info_p(gc, init)("Soft Max Capacity: " SIZE_FORMAT "M", soft_max_capacity / M);
   if (ZPageSizeMedium > 0) {
     log_info_p(gc, init)("Medium Page Size: " SIZE_FORMAT "M", ZPageSizeMedium / M);
   } else {
@@ -173,24 +222,22 @@ ZPageAllocator::ZPageAllocator(ZWorkers* workers,
   // Check if uncommit should and can be enabled
   _physical.try_enable_uncommit(min_capacity, max_capacity);
 
-  // Pre-map initial capacity
-  if (!prime_cache(workers, initial_capacity)) {
-    log_error_p(gc)("Failed to allocate initial Java heap (" SIZE_FORMAT "M)", initial_capacity / M);
-    return;
-  }
-
   // Successfully initialized
   _initialized = true;
 }
 
+bool ZPageAllocator::is_initialized() const {
+  return _initialized;
+}
+
 class ZPreTouchTask : public ZTask {
 private:
   const ZPhysicalMemoryManager* const _physical;
-  volatile uintptr_t                  _start;
-  const uintptr_t                     _end;
+  volatile zoffset                    _start;
+  const zoffset_end                   _end;
 
 public:
-  ZPreTouchTask(const ZPhysicalMemoryManager* physical, uintptr_t start, uintptr_t end) :
+  ZPreTouchTask(const ZPhysicalMemoryManager* physical, zoffset start, zoffset_end end) :
       ZTask("ZPreTouchTask"),
       _physical(physical),
       _start(start),
@@ -200,7 +247,7 @@ public:
     for (;;) {
       // Get granule offset
       const size_t size = ZGranuleSize;
-      const uintptr_t offset = Atomic::fetch_and_add(&_start, size);
+      const zoffset offset = to_zoffset(Atomic::fetch_and_add((uintptr_t*)&_start, size));
       if (offset >= _end) {
         // Done
         break;
@@ -214,12 +261,11 @@ public:
 
 bool ZPageAllocator::prime_cache(ZWorkers* workers, size_t size) {
   ZAllocationFlags flags;
-
   flags.set_non_blocking();
   flags.set_low_address();
 
-  ZPage* const page = alloc_page(ZPageTypeLarge, size, flags);
-  if (page == NULL) {
+  ZPage* const page = alloc_page(ZPageType::large, size, flags, ZPageAge::eden);
+  if (page == nullptr) {
     return false;
   }
 
@@ -229,13 +275,13 @@ bool ZPageAllocator::prime_cache(ZWorkers* workers, size_t size) {
     workers->run_all(&task);
   }
 
-  free_page(page, false /* reclaimed */);
+  free_page(page);
 
   return true;
 }
 
-bool ZPageAllocator::is_initialized() const {
-  return _initialized;
+size_t ZPageAllocator::initial_capacity() const {
+  return _initial_capacity;
 }
 
 size_t ZPageAllocator::min_capacity() const {
@@ -261,6 +307,10 @@ size_t ZPageAllocator::used() const {
   return Atomic::load(&_used);
 }
 
+size_t ZPageAllocator::used_generation(ZGenerationId id) const {
+  return Atomic::load(&_used_generations[(int)id]);
+}
+
 size_t ZPageAllocator::unused() const {
   const ssize_t capacity = (ssize_t)Atomic::load(&_capacity);
   const ssize_t used = (ssize_t)Atomic::load(&_used);
@@ -269,23 +319,26 @@ size_t ZPageAllocator::unused() const {
   return unused > 0 ? (size_t)unused : 0;
 }
 
-ZPageAllocatorStats ZPageAllocator::stats() const {
+ZPageAllocatorStats ZPageAllocator::stats(ZGeneration* generation) const {
   ZLocker<ZLock> locker(&_lock);
   return ZPageAllocatorStats(_min_capacity,
                              _max_capacity,
                              soft_max_capacity(),
                              _capacity,
                              _used,
-                             _used_high,
-                             _used_low,
-                             _reclaimed);
+                             _collection_stats[(int)generation->id()]._used_high,
+                             _collection_stats[(int)generation->id()]._used_low,
+                             used_generation(generation->id()),
+                             generation->freed(),
+                             generation->promoted(),
+                             generation->compacted(),
+                             _stalled.size());
 }
 
-void ZPageAllocator::reset_statistics() {
+void ZPageAllocator::reset_statistics(ZGenerationId id) {
   assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
-  _reclaimed = 0;
-  _used_high = _used_low = _used;
-  _nstalled = 0;
+  _collection_stats[(int)id]._used_high = _used;
+  _collection_stats[(int)id]._used_low = _used;
 }
 
 size_t ZPageAllocator::increase_capacity(size_t size) {
@@ -322,36 +375,53 @@ void ZPageAllocator::decrease_capacity(size_t size, bool set_max_capacity) {
   }
 }
 
-void ZPageAllocator::increase_used(size_t size, bool worker_relocation) {
-  if (worker_relocation) {
-    // Allocating a page for the purpose of worker relocation has
-    // a negative contribution to the number of reclaimed bytes.
-    _reclaimed -= size;
-  }
+void ZPageAllocator::increase_used(size_t size) {
+  // We don't track generation usage here because this page
+  // could be allocated by a thread that satisfies a stalling
+  // allocation. The stalled thread can wake up and potentially
+  // realize that the page alloc should be undone. If the alloc
+  // and the undo gets separated by a safepoint, the generation
+  // statistics could se a decreasing used value between mark
+  // start and mark end.
 
   // Update atomically since we have concurrent readers
   const size_t used = Atomic::add(&_used, size);
-  if (used > _used_high) {
-    _used_high = used;
+
+  // Update used high
+  for (auto& stats : _collection_stats) {
+    if (used > stats._used_high) {
+      stats._used_high = used;
+    }
   }
 }
 
-void ZPageAllocator::decrease_used(size_t size, bool reclaimed) {
-  // Only pages explicitly released with the reclaimed flag set
-  // counts as reclaimed bytes. This flag is true when we release
-  // a page after relocation, and is false when we release a page
-  // to undo an allocation.
-  if (reclaimed) {
-    _reclaimed += size;
-  }
-
+void ZPageAllocator::decrease_used(size_t size) {
   // Update atomically since we have concurrent readers
   const size_t used = Atomic::sub(&_used, size);
-  if (used < _used_low) {
-    _used_low = used;
+
+  // Update used low
+  for (auto& stats : _collection_stats) {
+    if (used < stats._used_low) {
+      stats._used_low = used;
+    }
   }
 }
 
+void ZPageAllocator::increase_used_generation(ZGenerationId id, size_t size) {
+  // Update atomically since we have concurrent readers
+  Atomic::add(&_used_generations[(int)id], size, memory_order_relaxed);
+}
+
+void ZPageAllocator::decrease_used_generation(ZGenerationId id, size_t size) {
+  // Update atomically since we have concurrent readers
+  Atomic::sub(&_used_generations[(int)id], size, memory_order_relaxed);
+}
+
+void ZPageAllocator::promote_used(size_t size) {
+  decrease_used_generation(ZGenerationId::young, size);
+  increase_used_generation(ZGenerationId::old, size);
+}
+
 bool ZPageAllocator::commit_page(ZPage* page) {
   // Commit physical memory
   return _physical.commit(page->physical_memory());
@@ -376,6 +446,11 @@ void ZPageAllocator::unmap_page(const ZPage* page) const {
   _physical.unmap(page->start(), page->size());
 }
 
+void ZPageAllocator::safe_destroy_page(ZPage* page) {
+  // Destroy page safely
+  _safe_destroy.schedule_delete(page);
+}
+
 void ZPageAllocator::destroy_page(ZPage* page) {
   // Free virtual memory
   _virtual.free(page->virtual_memory());
@@ -383,8 +458,8 @@ void ZPageAllocator::destroy_page(ZPage* page) {
   // Free physical memory
   _physical.free(page->physical_memory());
 
-  // Delete page safely
-  _safe_delete(page);
+  // Destroy page safely
+  safe_destroy_page(page);
 }
 
 bool ZPageAllocator::is_alloc_allowed(size_t size) const {
@@ -392,7 +467,7 @@ bool ZPageAllocator::is_alloc_allowed(size_t size) const {
   return available >= size;
 }
 
-bool ZPageAllocator::alloc_page_common_inner(uint8_t type, size_t size, ZList<ZPage>* pages) {
+bool ZPageAllocator::alloc_page_common_inner(ZPageType type, size_t size, ZList<ZPage>* pages) {
   if (!is_alloc_allowed(size)) {
     // Out of memory
     return false;
@@ -400,7 +475,7 @@ bool ZPageAllocator::alloc_page_common_inner(uint8_t type, size_t size, ZList<ZP
 
   // Try allocate from the page cache
   ZPage* const page = _cache.alloc_page(type, size);
-  if (page != NULL) {
+  if (page != nullptr) {
     // Success
     pages->insert_last(page);
     return true;
@@ -420,7 +495,7 @@ bool ZPageAllocator::alloc_page_common_inner(uint8_t type, size_t size, ZList<ZP
 }
 
 bool ZPageAllocator::alloc_page_common(ZPageAllocation* allocation) {
-  const uint8_t type = allocation->type();
+  const ZPageType type = allocation->type();
   const size_t size = allocation->size();
   const ZAllocationFlags flags = allocation->flags();
   ZList<ZPage>* const pages = allocation->pages();
@@ -431,7 +506,7 @@ bool ZPageAllocator::alloc_page_common(ZPageAllocation* allocation) {
   }
 
   // Updated used statistics
-  increase_used(size, flags.worker_relocation());
+  increase_used(size);
 
   // Success
   return true;
@@ -446,44 +521,32 @@ static void check_out_of_memory_during_initialization() {
 bool ZPageAllocator::alloc_page_stall(ZPageAllocation* allocation) {
   ZStatTimer timer(ZCriticalPhaseAllocationStall);
   EventZAllocationStall event;
-  ZPageAllocationStall result;
 
   // We can only block if the VM is fully initialized
   check_out_of_memory_during_initialization();
 
-  // Increment stalled counter
-  Atomic::inc(&_nstalled);
+  // Start asynchronous minor GC
+  const ZDriverRequest request(GCCause::_z_allocation_stall, ZYoungGCThreads, 0);
+  ZDriver::minor()->collect(request);
 
-  do {
-    // Start asynchronous GC
-    ZCollectedHeap::heap()->collect(GCCause::_z_allocation_stall);
-
-    // Wait for allocation to complete, fail or request a GC
-    result = allocation->wait();
-  } while (result == ZPageAllocationStallStartGC);
+  // Wait for allocation to complete or fail
+  const bool result = allocation->wait();
 
   {
-    //
-    // We grab the lock here for two different reasons:
-    //
-    // 1) Guard deletion of underlying semaphore. This is a workaround for
+    // Guard deletion of underlying semaphore. This is a workaround for
     // a bug in sem_post() in glibc < 2.21, where it's not safe to destroy
     // the semaphore immediately after returning from sem_wait(). The
     // reason is that sem_post() can touch the semaphore after a waiting
     // thread have returned from sem_wait(). To avoid this race we are
     // forcing the waiting thread to acquire/release the lock held by the
     // posting thread. https://sourceware.org/bugzilla/show_bug.cgi?id=12674
-    //
-    // 2) Guard the list of satisfied pages.
-    //
     ZLocker<ZLock> locker(&_lock);
-    _satisfied.remove(allocation);
   }
 
   // Send event
-  event.commit(allocation->type(), allocation->size());
+  event.commit((u8)allocation->type(), allocation->size());
 
-  return (result == ZPageAllocationStallSuccess);
+  return result;
 }
 
 bool ZPageAllocator::alloc_page_or_stall(ZPageAllocation* allocation) {
@@ -519,7 +582,7 @@ ZPage* ZPageAllocator::alloc_page_create(ZPageAllocation* allocation) {
   const ZVirtualMemory vmem = _virtual.alloc(size, allocation->flags().low_address());
   if (vmem.is_null()) {
     log_error(gc)("Out of address space");
-    return NULL;
+    return nullptr;
   }
 
   ZPhysicalMemory pmem;
@@ -565,8 +628,8 @@ bool ZPageAllocator::should_defragment(const ZPage* page) const {
   // if we've split a larger page or we have a constrained address space. To help
   // fight address space fragmentation we remap such pages to a lower address, if
   // a lower address is available.
-  return page->type() == ZPageTypeSmall &&
-         page->start() >= _virtual.reserved() / 2 &&
+  return page->type() == ZPageType::small &&
+         page->start() >= to_zoffset(_virtual.reserved() / 2) &&
          page->start() > _virtual.lowest_available_address();
 }
 
@@ -607,9 +670,9 @@ ZPage* ZPageAllocator::alloc_page_finalize(ZPageAllocation* allocation) {
 
   // Slow path
   ZPage* const page = alloc_page_create(allocation);
-  if (page == NULL) {
+  if (page == nullptr) {
     // Out of address space
-    return NULL;
+    return nullptr;
   }
 
   // Commit page
@@ -625,36 +688,15 @@ ZPage* ZPageAllocator::alloc_page_finalize(ZPageAllocation* allocation) {
   ZPage* const committed_page = page->split_committed();
   destroy_page(page);
 
-  if (committed_page != NULL) {
+  if (committed_page != nullptr) {
     map_page(committed_page);
     allocation->pages()->insert_last(committed_page);
   }
 
-  return NULL;
+  return nullptr;
 }
 
-void ZPageAllocator::alloc_page_failed(ZPageAllocation* allocation) {
-  ZLocker<ZLock> locker(&_lock);
-
-  size_t freed = 0;
-
-  // Free any allocated/flushed pages
-  ZListRemoveIterator<ZPage> iter(allocation->pages());
-  for (ZPage* page; iter.next(&page);) {
-    freed += page->size();
-    free_page_inner(page, false /* reclaimed */);
-  }
-
-  // Adjust capacity and used to reflect the failed capacity increase
-  const size_t remaining = allocation->size() - freed;
-  decrease_used(remaining, false /* reclaimed */);
-  decrease_capacity(remaining, true /* set_max_capacity */);
-
-  // Try satisfy stalled allocations
-  satisfy_stalled();
-}
-
-ZPage* ZPageAllocator::alloc_page(uint8_t type, size_t size, ZAllocationFlags flags) {
+ZPage* ZPageAllocator::alloc_page(ZPageType type, size_t size, ZAllocationFlags flags, ZPageAge age) {
   EventZPageAllocation event;
 
 retry:
@@ -667,34 +709,39 @@ retry:
   // block in a safepoint if the non-blocking flag is not set.
   if (!alloc_page_or_stall(&allocation)) {
     // Out of memory
-    return NULL;
+    return nullptr;
   }
 
   ZPage* const page = alloc_page_finalize(&allocation);
-  if (page == NULL) {
+  if (page == nullptr) {
     // Failed to commit or map. Clean up and retry, in the hope that
     // we can still allocate by flushing the page cache (more aggressively).
-    alloc_page_failed(&allocation);
+    free_pages_alloc_failed(&allocation);
     goto retry;
   }
 
+  // The generation's used is tracked here when the page is handed out
+  // to the allocating thread. The overall heap "used" is tracked in
+  // the lower-level allocation code.
+  const ZGenerationId id = age == ZPageAge::old ? ZGenerationId::old : ZGenerationId::young;
+  increase_used_generation(id, size);
+
   // Reset page. This updates the page's sequence number and must
   // be done after we potentially blocked in a safepoint (stalled)
   // where the global sequence number was updated.
-  page->reset();
+  page->reset(age, ZPageResetType::Allocation);
 
-  // Update allocation statistics. Exclude worker relocations to avoid
+  // Update allocation statistics. Exclude gc relocations to avoid
   // artificial inflation of the allocation rate during relocation.
-  if (!flags.worker_relocation() && is_init_completed()) {
+  if (!flags.gc_relocation() && is_init_completed()) {
     // Note that there are two allocation rate counters, which have
     // different purposes and are sampled at different frequencies.
-    const size_t bytes = page->size();
-    ZStatInc(ZCounterAllocationRate, bytes);
-    ZStatInc(ZStatAllocRate::counter(), bytes);
+    ZStatInc(ZCounterMutatorAllocationRate, size);
+    ZStatMutatorAllocRate::sample_allocation(size);
   }
 
   // Send event
-  event.commit(type, size, allocation.flushed(), allocation.committed(),
+  event.commit((u8)type, size, allocation.flushed(), allocation.committed(),
                page->physical_memory().nsegments(), flags.non_blocking());
 
   return page;
@@ -703,7 +750,7 @@ retry:
 void ZPageAllocator::satisfy_stalled() {
   for (;;) {
     ZPageAllocation* const allocation = _stalled.first();
-    if (allocation == NULL) {
+    if (allocation == nullptr) {
       // Allocation queue is empty
       return;
     }
@@ -717,15 +764,11 @@ void ZPageAllocator::satisfy_stalled() {
     // Note that we must dequeue the allocation request first, since
     // it will immediately be deallocated once it has been satisfied.
     _stalled.remove(allocation);
-    _satisfied.insert_last(allocation);
-    allocation->satisfy(ZPageAllocationStallSuccess);
+    allocation->satisfy(true);
   }
 }
 
-void ZPageAllocator::free_page_inner(ZPage* page, bool reclaimed) {
-  // Update used statistics
-  decrease_used(page->size(), reclaimed);
-
+void ZPageAllocator::recycle_page(ZPage* page) {
   // Set time when last used
   page->set_last_used();
 
@@ -733,41 +776,96 @@ void ZPageAllocator::free_page_inner(ZPage* page, bool reclaimed) {
   _cache.free_page(page);
 }
 
-void ZPageAllocator::free_page(ZPage* page, bool reclaimed) {
+void ZPageAllocator::free_page(ZPage* page) {
+  const ZGenerationId generation_id = page->generation_id();
+  ZPage* const to_recycle = _safe_recycle.register_and_clone_if_activated(page);
+
   ZLocker<ZLock> locker(&_lock);
 
+  // Update used statistics
+  const size_t size = to_recycle->size();
+  decrease_used(size);
+  decrease_used_generation(generation_id, size);
+
   // Free page
-  free_page_inner(page, reclaimed);
+  recycle_page(to_recycle);
 
   // Try satisfy stalled allocations
   satisfy_stalled();
 }
 
-void ZPageAllocator::free_pages(const ZArray<ZPage*>* pages, bool reclaimed) {
+void ZPageAllocator::free_pages(const ZArray<ZPage*>* pages) {
+  ZArray<ZPage*> to_recycle;
+
+  size_t young_size = 0;
+  size_t old_size = 0;
+
+  ZArrayIterator<ZPage*> pages_iter(pages);
+  for (ZPage* page; pages_iter.next(&page);) {
+    if (page->is_young()) {
+      young_size += page->size();
+    } else {
+      old_size += page->size();
+    }
+    to_recycle.push(_safe_recycle.register_and_clone_if_activated(page));
+  }
+
   ZLocker<ZLock> locker(&_lock);
 
+  // Update used statistics
+  decrease_used(young_size + old_size);
+  decrease_used_generation(ZGenerationId::young, young_size);
+  decrease_used_generation(ZGenerationId::old, old_size);
+
   // Free pages
-  ZArrayIterator<ZPage*> iter(pages);
+  ZArrayIterator<ZPage*> iter(&to_recycle);
   for (ZPage* page; iter.next(&page);) {
-    free_page_inner(page, reclaimed);
+    recycle_page(page);
   }
 
   // Try satisfy stalled allocations
   satisfy_stalled();
 }
 
+void ZPageAllocator::free_pages_alloc_failed(ZPageAllocation* allocation) {
+  ZArray<ZPage*> to_recycle;
+
+  ZListRemoveIterator<ZPage> allocation_pages_iter(allocation->pages());
+  for (ZPage* page; allocation_pages_iter.next(&page);) {
+    to_recycle.push(_safe_recycle.register_and_clone_if_activated(page));
+  }
+
+  ZLocker<ZLock> locker(&_lock);
+
+  // Only decrease the overall used and not the generation used,
+  // since the allocation failed and generation used wasn't bumped.
+  decrease_used(allocation->size());
+
+  size_t freed = 0;
+
+  // Free any allocated/flushed pages
+  ZArrayIterator<ZPage*> iter(&to_recycle);
+  for (ZPage* page; iter.next(&page);) {
+    freed += page->size();
+    recycle_page(page);
+  }
+
+  // Adjust capacity and used to reflect the failed capacity increase
+  const size_t remaining = allocation->size() - freed;
+  decrease_capacity(remaining, true /* set_max_capacity */);
+
+  // Try satisfy stalled allocations
+  satisfy_stalled();
+}
+
 size_t ZPageAllocator::uncommit(uint64_t* timeout) {
   // We need to join the suspendible thread set while manipulating capacity and
-  // used, to make sure GC safepoints will have a consistent view. However, when
-  // ZVerifyViews is enabled we need to join at a broader scope to also make sure
-  // we don't change the address good mask after pages have been flushed, and
-  // thereby made invisible to pages_do(), but before they have been unmapped.
-  SuspendibleThreadSetJoiner joiner(ZVerifyViews);
+  // used, to make sure GC safepoints will have a consistent view.
   ZList<ZPage> pages;
   size_t flushed;
 
   {
-    SuspendibleThreadSetJoiner joiner(!ZVerifyViews);
+    SuspendibleThreadSetJoiner sts_joiner;
     ZLocker<ZLock> locker(&_lock);
 
     // Never uncommit below min capacity. We flush out and uncommit chunks at
@@ -798,7 +896,7 @@ size_t ZPageAllocator::uncommit(uint64_t* timeout) {
   }
 
   {
-    SuspendibleThreadSetJoiner joiner(!ZVerifyViews);
+    SuspendibleThreadSetJoiner sts_joiner;
     ZLocker<ZLock> locker(&_lock);
 
     // Adjust claimed and capacity to reflect the uncommit
@@ -809,61 +907,90 @@ size_t ZPageAllocator::uncommit(uint64_t* timeout) {
   return flushed;
 }
 
-void ZPageAllocator::enable_deferred_delete() const {
-  _safe_delete.enable_deferred_delete();
+void ZPageAllocator::enable_safe_destroy() const {
+  _safe_destroy.enable_deferred_delete();
 }
 
-void ZPageAllocator::disable_deferred_delete() const {
-  _safe_delete.disable_deferred_delete();
+void ZPageAllocator::disable_safe_destroy() const {
+  _safe_destroy.disable_deferred_delete();
 }
 
-void ZPageAllocator::debug_map_page(const ZPage* page) const {
-  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
-  _physical.debug_map(page->start(), page->physical_memory());
+void ZPageAllocator::enable_safe_recycle() const {
+  _safe_recycle.activate();
 }
 
-void ZPageAllocator::debug_unmap_page(const ZPage* page) const {
-  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
-  _physical.debug_unmap(page->start(), page->size());
+void ZPageAllocator::disable_safe_recycle() const {
+  _safe_recycle.deactivate();
 }
 
-void ZPageAllocator::pages_do(ZPageClosure* cl) const {
-  assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
-
-  ZListIterator<ZPageAllocation> iter_satisfied(&_satisfied);
-  for (ZPageAllocation* allocation; iter_satisfied.next(&allocation);) {
-    ZListIterator<ZPage> iter_pages(allocation->pages());
-    for (ZPage* page; iter_pages.next(&page);) {
-      cl->do_page(page);
-    }
-  }
-
-  _cache.pages_do(cl);
+static bool has_alloc_seen_young(const ZPageAllocation* allocation) {
+  return allocation->young_seqnum() != ZGeneration::young()->seqnum();
 }
 
-bool ZPageAllocator::has_alloc_stalled() const {
-  return Atomic::load(&_nstalled) != 0;
+static bool has_alloc_seen_old(const ZPageAllocation* allocation) {
+  return allocation->old_seqnum() != ZGeneration::old()->seqnum();
 }
 
-void ZPageAllocator::check_out_of_memory() {
+bool ZPageAllocator::is_alloc_stalling() const {
+  ZLocker<ZLock> locker(&_lock);
+  return _stalled.first() != nullptr;
+}
+
+bool ZPageAllocator::is_alloc_stalling_for_old() const {
   ZLocker<ZLock> locker(&_lock);
 
-  // Fail allocation requests that were enqueued before the
-  // last GC cycle started, otherwise start a new GC cycle.
-  for (ZPageAllocation* allocation = _stalled.first(); allocation != NULL; allocation = _stalled.first()) {
-    if (allocation->seqnum() == ZGlobalSeqNum) {
-      // Start a new GC cycle, keep allocation requests enqueued
-      allocation->satisfy(ZPageAllocationStallStartGC);
+  ZPageAllocation* const allocation = _stalled.first();
+  if (allocation == nullptr) {
+    // No stalled allocations
+    return false;
+  }
+
+  return has_alloc_seen_young(allocation) && !has_alloc_seen_old(allocation);
+}
+
+void ZPageAllocator::notify_out_of_memory() {
+  // Fail allocation requests that were enqueued before the last major GC started
+  for (ZPageAllocation* allocation = _stalled.first(); allocation != nullptr; allocation = _stalled.first()) {
+    if (!has_alloc_seen_old(allocation)) {
+      // Not out of memory, keep remaining allocation requests enqueued
       return;
     }
 
-    // Out of memory, fail allocation request
+    // Out of memory, dequeue and fail allocation request
     _stalled.remove(allocation);
-    _satisfied.insert_last(allocation);
-    allocation->satisfy(ZPageAllocationStallFailed);
+    allocation->satisfy(false);
   }
 }
 
+void ZPageAllocator::restart_gc() const {
+  ZPageAllocation* const allocation = _stalled.first();
+  if (allocation == nullptr) {
+    // No stalled allocations
+    return;
+  }
+
+  if (!has_alloc_seen_young(allocation)) {
+    // Start asynchronous minor GC, keep allocation requests enqueued
+    const ZDriverRequest request(GCCause::_z_allocation_stall, ZYoungGCThreads, 0);
+    ZDriver::minor()->collect(request);
+  } else {
+    // Start asynchronous major GC, keep allocation requests enqueued
+    const ZDriverRequest request(GCCause::_z_allocation_stall, ZYoungGCThreads, ZOldGCThreads);
+    ZDriver::major()->collect(request);
+  }
+}
+
+void ZPageAllocator::handle_alloc_stalling_for_young() {
+  ZLocker<ZLock> locker(&_lock);
+  restart_gc();
+}
+
+void ZPageAllocator::handle_alloc_stalling_for_old() {
+  ZLocker<ZLock> locker(&_lock);
+  notify_out_of_memory();
+  restart_gc();
+}
+
 void ZPageAllocator::threads_do(ThreadClosure* tc) const {
   tc->do_thread(_unmapper);
   tc->do_thread(_uncommitter);
diff --git a/src/hotspot/share/gc/z/zPageAllocator.hpp b/src/hotspot/share/gc/z/zPageAllocator.hpp
index 963259977af..3aaaeb41f2c 100644
--- a/src/hotspot/share/gc/z/zPageAllocator.hpp
+++ b/src/hotspot/share/gc/z/zPageAllocator.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,18 +28,36 @@
 #include "gc/z/zArray.hpp"
 #include "gc/z/zList.hpp"
 #include "gc/z/zLock.hpp"
+#include "gc/z/zPageAge.hpp"
 #include "gc/z/zPageCache.hpp"
+#include "gc/z/zPageType.hpp"
 #include "gc/z/zPhysicalMemory.hpp"
 #include "gc/z/zSafeDelete.hpp"
 #include "gc/z/zVirtualMemory.hpp"
 
 class ThreadClosure;
+class ZGeneration;
 class ZPageAllocation;
+class ZPageAllocator;
 class ZPageAllocatorStats;
 class ZWorkers;
 class ZUncommitter;
 class ZUnmapper;
 
+class ZSafePageRecycle {
+private:
+  ZPageAllocator*        _page_allocator;
+  ZActivatedArray<ZPage> _unsafe_to_recycle;
+
+public:
+  ZSafePageRecycle(ZPageAllocator* page_allocator);
+
+  void activate();
+  void deactivate();
+
+  ZPage* register_and_clone_if_activated(ZPage* page);
+};
+
 class ZPageAllocator {
   friend class VMStructs;
   friend class ZUnmapper;
@@ -51,29 +69,32 @@ private:
   ZVirtualMemoryManager      _virtual;
   ZPhysicalMemoryManager     _physical;
   const size_t               _min_capacity;
+  const size_t               _initial_capacity;
   const size_t               _max_capacity;
   volatile size_t            _current_max_capacity;
   volatile size_t            _capacity;
   volatile size_t            _claimed;
   volatile size_t            _used;
-  size_t                     _used_high;
-  size_t                     _used_low;
-  ssize_t                    _reclaimed;
+  size_t                     _used_generations[2];
+  struct {
+    size_t                   _used_high;
+    size_t                   _used_low;
+  } _collection_stats[2];
   ZList<ZPageAllocation>     _stalled;
-  volatile uint64_t          _nstalled;
-  ZList<ZPageAllocation>     _satisfied;
   ZUnmapper*                 _unmapper;
   ZUncommitter*              _uncommitter;
-  mutable ZSafeDelete<ZPage> _safe_delete;
+  mutable ZSafeDelete<ZPage> _safe_destroy;
+  mutable ZSafePageRecycle   _safe_recycle;
   bool                       _initialized;
 
-  bool prime_cache(ZWorkers* workers, size_t size);
-
   size_t increase_capacity(size_t size);
   void decrease_capacity(size_t size, bool set_max_capacity);
 
-  void increase_used(size_t size, bool relocation);
-  void decrease_used(size_t size, bool reclaimed);
+  void increase_used(size_t size);
+  void decrease_used(size_t size);
+
+  void increase_used_generation(ZGenerationId id, size_t size);
+  void decrease_used_generation(ZGenerationId id, size_t size);
 
   bool commit_page(ZPage* page);
   void uncommit_page(ZPage* page);
@@ -85,7 +106,7 @@ private:
 
   bool is_alloc_allowed(size_t size) const;
 
-  bool alloc_page_common_inner(uint8_t type, size_t size, ZList<ZPage>* pages);
+  bool alloc_page_common_inner(ZPageType type, size_t size, ZList<ZPage>* pages);
   bool alloc_page_common(ZPageAllocation* allocation);
   bool alloc_page_stall(ZPageAllocation* allocation);
   bool alloc_page_or_stall(ZPageAllocation* allocation);
@@ -93,47 +114,56 @@ private:
   bool is_alloc_satisfied(ZPageAllocation* allocation) const;
   ZPage* alloc_page_create(ZPageAllocation* allocation);
   ZPage* alloc_page_finalize(ZPageAllocation* allocation);
-  void alloc_page_failed(ZPageAllocation* allocation);
+  void free_pages_alloc_failed(ZPageAllocation* allocation);
 
   void satisfy_stalled();
 
-  void free_page_inner(ZPage* page, bool reclaimed);
-
   size_t uncommit(uint64_t* timeout);
 
+  void notify_out_of_memory();
+  void restart_gc() const;
+
 public:
-  ZPageAllocator(ZWorkers* workers,
-                 size_t min_capacity,
+  ZPageAllocator(size_t min_capacity,
                  size_t initial_capacity,
+                 size_t soft_max_capacity,
                  size_t max_capacity);
 
   bool is_initialized() const;
 
+  bool prime_cache(ZWorkers* workers, size_t size);
+
+  size_t initial_capacity() const;
   size_t min_capacity() const;
   size_t max_capacity() const;
   size_t soft_max_capacity() const;
   size_t capacity() const;
   size_t used() const;
+  size_t used_generation(ZGenerationId id) const;
   size_t unused() const;
 
-  ZPageAllocatorStats stats() const;
+  void promote_used(size_t size);
 
-  void reset_statistics();
+  ZPageAllocatorStats stats(ZGeneration* generation) const;
 
-  ZPage* alloc_page(uint8_t type, size_t size, ZAllocationFlags flags);
-  void free_page(ZPage* page, bool reclaimed);
-  void free_pages(const ZArray<ZPage*>* pages, bool reclaimed);
+  void reset_statistics(ZGenerationId id);
 
-  void enable_deferred_delete() const;
-  void disable_deferred_delete() const;
+  ZPage* alloc_page(ZPageType type, size_t size, ZAllocationFlags flags, ZPageAge age);
+  void recycle_page(ZPage* page);
+  void safe_destroy_page(ZPage* page);
+  void free_page(ZPage* page);
+  void free_pages(const ZArray<ZPage*>* pages);
 
-  void debug_map_page(const ZPage* page) const;
-  void debug_unmap_page(const ZPage* page) const;
+  void enable_safe_destroy() const;
+  void disable_safe_destroy() const;
 
-  bool has_alloc_stalled() const;
-  void check_out_of_memory();
+  void enable_safe_recycle() const;
+  void disable_safe_recycle() const;
 
-  void pages_do(ZPageClosure* cl) const;
+  bool is_alloc_stalling() const;
+  bool is_alloc_stalling_for_old() const;
+  void handle_alloc_stalling_for_young();
+  void handle_alloc_stalling_for_old();
 
   void threads_do(ThreadClosure* tc) const;
 };
@@ -143,12 +173,15 @@ private:
   size_t _min_capacity;
   size_t _max_capacity;
   size_t _soft_max_capacity;
-  size_t _current_max_capacity;
   size_t _capacity;
   size_t _used;
   size_t _used_high;
   size_t _used_low;
-  size_t _reclaimed;
+  size_t _used_generation;
+  size_t _freed;
+  size_t _promoted;
+  size_t _compacted;
+  size_t _allocation_stalls;
 
 public:
   ZPageAllocatorStats(size_t min_capacity,
@@ -158,7 +191,11 @@ public:
                       size_t used,
                       size_t used_high,
                       size_t used_low,
-                      size_t reclaimed);
+                      size_t used_generation,
+                      size_t freed,
+                      size_t promoted,
+                      size_t compacted,
+                      size_t allocation_stalls);
 
   size_t min_capacity() const;
   size_t max_capacity() const;
@@ -167,7 +204,11 @@ public:
   size_t used() const;
   size_t used_high() const;
   size_t used_low() const;
-  size_t reclaimed() const;
+  size_t used_generation() const;
+  size_t freed() const;
+  size_t promoted() const;
+  size_t compacted() const;
+  size_t allocation_stalls() const;
 };
 
 #endif // SHARE_GC_Z_ZPAGEALLOCATOR_HPP
diff --git a/src/hotspot/share/gc/z/zPageAllocator.inline.hpp b/src/hotspot/share/gc/z/zPageAllocator.inline.hpp
index 881765ee2a4..539fca128b6 100644
--- a/src/hotspot/share/gc/z/zPageAllocator.inline.hpp
+++ b/src/hotspot/share/gc/z/zPageAllocator.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -33,7 +33,11 @@ inline ZPageAllocatorStats::ZPageAllocatorStats(size_t min_capacity,
                                                 size_t used,
                                                 size_t used_high,
                                                 size_t used_low,
-                                                size_t reclaimed) :
+                                                size_t used_generation,
+                                                size_t freed,
+                                                size_t promoted,
+                                                size_t compacted,
+                                                size_t allocation_stalls) :
     _min_capacity(min_capacity),
     _max_capacity(max_capacity),
     _soft_max_capacity(soft_max_capacity),
@@ -41,7 +45,11 @@ inline ZPageAllocatorStats::ZPageAllocatorStats(size_t min_capacity,
     _used(used),
     _used_high(used_high),
     _used_low(used_low),
-    _reclaimed(reclaimed) {}
+    _used_generation(used_generation),
+    _freed(freed),
+    _promoted(promoted),
+    _compacted(compacted),
+    _allocation_stalls(allocation_stalls) {}
 
 inline size_t ZPageAllocatorStats::min_capacity() const {
   return _min_capacity;
@@ -71,8 +79,24 @@ inline size_t ZPageAllocatorStats::used_low() const {
   return _used_low;
 }
 
-inline size_t ZPageAllocatorStats::reclaimed() const {
-  return _reclaimed;
+inline size_t ZPageAllocatorStats::used_generation() const {
+  return _used_generation;
+}
+
+inline size_t ZPageAllocatorStats::freed() const {
+  return _freed;
+}
+
+inline size_t ZPageAllocatorStats::promoted() const {
+  return _promoted;
+}
+
+inline size_t ZPageAllocatorStats::compacted() const {
+  return _compacted;
+}
+
+inline size_t ZPageAllocatorStats::allocation_stalls() const {
+  return _allocation_stalls;
 }
 
 #endif // SHARE_GC_Z_ZPAGEALLOCATOR_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zPageCache.cpp b/src/hotspot/share/gc/z/zPageCache.cpp
index ae05b257244..04fbb574728 100644
--- a/src/hotspot/share/gc/z/zPageCache.cpp
+++ b/src/hotspot/share/gc/z/zPageCache.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -66,7 +66,7 @@ ZPage* ZPageCache::alloc_small_page() {
 
   // Try NUMA local page cache
   ZPage* const l1_page = _small.get(numa_id).remove_first();
-  if (l1_page != NULL) {
+  if (l1_page != nullptr) {
     ZStatInc(ZCounterPageCacheHitL1);
     return l1_page;
   }
@@ -80,7 +80,7 @@ ZPage* ZPageCache::alloc_small_page() {
     }
 
     ZPage* const l2_page = _small.get(remote_numa_id).remove_first();
-    if (l2_page != NULL) {
+    if (l2_page != nullptr) {
       ZStatInc(ZCounterPageCacheHitL2);
       return l2_page;
     }
@@ -88,17 +88,17 @@ ZPage* ZPageCache::alloc_small_page() {
     remote_numa_id++;
   }
 
-  return NULL;
+  return nullptr;
 }
 
 ZPage* ZPageCache::alloc_medium_page() {
   ZPage* const page = _medium.remove_first();
-  if (page != NULL) {
+  if (page != nullptr) {
     ZStatInc(ZCounterPageCacheHitL1);
     return page;
   }
 
-  return NULL;
+  return nullptr;
 }
 
 ZPage* ZPageCache::alloc_large_page(size_t size) {
@@ -113,7 +113,7 @@ ZPage* ZPageCache::alloc_large_page(size_t size) {
     }
   }
 
-  return NULL;
+  return nullptr;
 }
 
 ZPage* ZPageCache::alloc_oversized_medium_page(size_t size) {
@@ -121,7 +121,7 @@ ZPage* ZPageCache::alloc_oversized_medium_page(size_t size) {
     return _medium.remove_first();
   }
 
-  return NULL;
+  return nullptr;
 }
 
 ZPage* ZPageCache::alloc_oversized_large_page(size_t size) {
@@ -135,38 +135,38 @@ ZPage* ZPageCache::alloc_oversized_large_page(size_t size) {
     }
   }
 
-  return NULL;
+  return nullptr;
 }
 
 ZPage* ZPageCache::alloc_oversized_page(size_t size) {
   ZPage* page = alloc_oversized_large_page(size);
-  if (page == NULL) {
+  if (page == nullptr) {
     page = alloc_oversized_medium_page(size);
   }
 
-  if (page != NULL) {
+  if (page != nullptr) {
     ZStatInc(ZCounterPageCacheHitL3);
   }
 
   return page;
 }
 
-ZPage* ZPageCache::alloc_page(uint8_t type, size_t size) {
+ZPage* ZPageCache::alloc_page(ZPageType type, size_t size) {
   ZPage* page;
 
   // Try allocate exact page
-  if (type == ZPageTypeSmall) {
+  if (type == ZPageType::small) {
     page = alloc_small_page();
-  } else if (type == ZPageTypeMedium) {
+  } else if (type == ZPageType::medium) {
     page = alloc_medium_page();
   } else {
     page = alloc_large_page(size);
   }
 
-  if (page == NULL) {
+  if (page == nullptr) {
     // Try allocate potentially oversized page
     ZPage* const oversized = alloc_oversized_page(size);
-    if (oversized != NULL) {
+    if (oversized != nullptr) {
       if (size < oversized->size()) {
         // Split oversized page
         page = oversized->split(type, size);
@@ -180,7 +180,7 @@ ZPage* ZPageCache::alloc_page(uint8_t type, size_t size) {
     }
   }
 
-  if (page == NULL) {
+  if (page == nullptr) {
     ZStatInc(ZCounterPageCacheMiss);
   }
 
@@ -188,10 +188,10 @@ ZPage* ZPageCache::alloc_page(uint8_t type, size_t size) {
 }
 
 void ZPageCache::free_page(ZPage* page) {
-  const uint8_t type = page->type();
-  if (type == ZPageTypeSmall) {
+  const ZPageType type = page->type();
+  if (type == ZPageType::small) {
     _small.get(page->numa_id()).insert_first(page);
-  } else if (type == ZPageTypeMedium) {
+  } else if (type == ZPageType::medium) {
     _medium.insert_first(page);
   } else {
     _large.insert_first(page);
@@ -200,7 +200,7 @@ void ZPageCache::free_page(ZPage* page) {
 
 bool ZPageCache::flush_list_inner(ZPageCacheFlushClosure* cl, ZList<ZPage>* from, ZList<ZPage>* to) {
   ZPage* const page = from->last();
-  if (page == NULL || !cl->do_page(page)) {
+  if (page == nullptr || !cl->do_page(page)) {
     // Don't flush page
     return false;
   }
@@ -222,7 +222,7 @@ void ZPageCache::flush_per_numa_lists(ZPageCacheFlushClosure* cl, ZPerNUMA<ZList
 
   // Flush lists round-robin
   while (numa_done < numa_count) {
-    ZList<ZPage>* numa_list = from->addr(numa_next);
+    ZList<ZPage>* const numa_list = from->addr(numa_next);
     if (++numa_next == numa_count) {
       numa_next = 0;
     }
@@ -331,26 +331,3 @@ size_t ZPageCache::flush_for_uncommit(size_t requested, ZList<ZPage>* to, uint64
 void ZPageCache::set_last_commit() {
   _last_commit = ceil(os::elapsedTime());
 }
-
-void ZPageCache::pages_do(ZPageClosure* cl) const {
-  // Small
-  ZPerNUMAConstIterator<ZList<ZPage> > iter_numa(&_small);
-  for (const ZList<ZPage>* list; iter_numa.next(&list);) {
-    ZListIterator<ZPage> iter_small(list);
-    for (ZPage* page; iter_small.next(&page);) {
-      cl->do_page(page);
-    }
-  }
-
-  // Medium
-  ZListIterator<ZPage> iter_medium(&_medium);
-  for (ZPage* page; iter_medium.next(&page);) {
-    cl->do_page(page);
-  }
-
-  // Large
-  ZListIterator<ZPage> iter_large(&_large);
-  for (ZPage* page; iter_large.next(&page);) {
-    cl->do_page(page);
-  }
-}
diff --git a/src/hotspot/share/gc/z/zPageCache.hpp b/src/hotspot/share/gc/z/zPageCache.hpp
index b641e0e4be1..b28aaa6c10d 100644
--- a/src/hotspot/share/gc/z/zPageCache.hpp
+++ b/src/hotspot/share/gc/z/zPageCache.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
 
 #include "gc/z/zList.hpp"
 #include "gc/z/zPage.hpp"
+#include "gc/z/zPageType.hpp"
 #include "gc/z/zValue.hpp"
 
 class ZPageCacheFlushClosure;
@@ -53,15 +54,13 @@ private:
 public:
   ZPageCache();
 
-  ZPage* alloc_page(uint8_t type, size_t size);
+  ZPage* alloc_page(ZPageType type, size_t size);
   void free_page(ZPage* page);
 
   void flush_for_allocation(size_t requested, ZList<ZPage>* to);
   size_t flush_for_uncommit(size_t requested, ZList<ZPage>* to, uint64_t* timeout);
 
   void set_last_commit();
-
-  void pages_do(ZPageClosure* cl) const;
 };
 
 #endif // SHARE_GC_Z_ZPAGECACHE_HPP
diff --git a/src/hotspot/share/gc/z/zPageTable.cpp b/src/hotspot/share/gc/z/zPageTable.cpp
index d462421097b..299f70eaad6 100644
--- a/src/hotspot/share/gc/z/zPageTable.cpp
+++ b/src/hotspot/share/gc/z/zPageTable.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,7 +22,7 @@
  */
 
 #include "precompiled.hpp"
-#include "gc/z/zGlobals.hpp"
+#include "gc/z/zAddress.hpp"
 #include "gc/z/zGranuleMap.inline.hpp"
 #include "gc/z/zPage.inline.hpp"
 #include "gc/z/zPageTable.inline.hpp"
@@ -33,21 +33,63 @@ ZPageTable::ZPageTable() :
     _map(ZAddressOffsetMax) {}
 
 void ZPageTable::insert(ZPage* page) {
-  const uintptr_t offset = page->start();
+  const zoffset offset = page->start();
   const size_t size = page->size();
 
   // Make sure a newly created page is
   // visible before updating the page table.
   OrderAccess::storestore();
 
-  assert(_map.get(offset) == NULL, "Invalid entry");
+  assert(_map.get(offset) == nullptr, "Invalid entry");
   _map.put(offset, size, page);
+
+  if (page->is_old()) {
+    ZGeneration::young()->register_with_remset(page);
+  }
 }
 
 void ZPageTable::remove(ZPage* page) {
-  const uintptr_t offset = page->start();
+  const zoffset offset = page->start();
   const size_t size = page->size();
 
   assert(_map.get(offset) == page, "Invalid entry");
-  _map.put(offset, size, NULL);
+  _map.put(offset, size, nullptr);
+}
+
+void ZPageTable::replace(ZPage* old_page, ZPage* new_page) {
+  const zoffset offset = old_page->start();
+  const size_t size = old_page->size();
+
+  assert(_map.get(offset) == old_page, "Invalid entry");
+  _map.release_put(offset, size, new_page);
+
+  if (new_page->is_old()) {
+    ZGeneration::young()->register_with_remset(new_page);
+  }
+}
+
+ZGenerationPagesParallelIterator::ZGenerationPagesParallelIterator(const ZPageTable* page_table, ZGenerationId id, ZPageAllocator* page_allocator) :
+    _iterator(page_table),
+    _generation_id(id),
+    _page_allocator(page_allocator) {
+  _page_allocator->enable_safe_destroy();
+  _page_allocator->enable_safe_recycle();
+}
+
+ZGenerationPagesParallelIterator::~ZGenerationPagesParallelIterator() {
+  _page_allocator->disable_safe_recycle();
+  _page_allocator->disable_safe_destroy();
+}
+
+ZGenerationPagesIterator::ZGenerationPagesIterator(const ZPageTable* page_table, ZGenerationId id, ZPageAllocator* page_allocator) :
+    _iterator(page_table),
+    _generation_id(id),
+    _page_allocator(page_allocator) {
+  _page_allocator->enable_safe_destroy();
+  _page_allocator->enable_safe_recycle();
+}
+
+ZGenerationPagesIterator::~ZGenerationPagesIterator() {
+  _page_allocator->disable_safe_recycle();
+  _page_allocator->disable_safe_destroy();
 }
diff --git a/src/hotspot/share/gc/z/zPageTable.hpp b/src/hotspot/share/gc/z/zPageTable.hpp
index fccb61b5843..e809a0ac0ca 100644
--- a/src/hotspot/share/gc/z/zPageTable.hpp
+++ b/src/hotspot/share/gc/z/zPageTable.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,14 +24,20 @@
 #ifndef SHARE_GC_Z_ZPAGETABLE_HPP
 #define SHARE_GC_Z_ZPAGETABLE_HPP
 
+#include "gc/z/zGenerationId.hpp"
 #include "gc/z/zGranuleMap.hpp"
+#include "gc/z/zIndexDistributor.hpp"
 #include "memory/allocation.hpp"
 
 class ZPage;
+class ZPageAllocator;
+class ZPageTable;
 
 class ZPageTable {
-  friend class VMStructs;
   friend class ZPageTableIterator;
+  friend class ZPageTableParallelIterator;
+  friend class ZRemsetTableIterator;
+  friend class VMStructs;
 
 private:
   ZGranuleMap<ZPage*> _map;
@@ -39,21 +45,66 @@ private:
 public:
   ZPageTable();
 
-  ZPage* get(uintptr_t addr) const;
+  ZPage* get(zaddress addr) const;
+  ZPage* get(volatile zpointer* p) const;
+
+  ZPage* at(size_t index) const;
 
   void insert(ZPage* page);
   void remove(ZPage* page);
+  void replace(ZPage* old_page, ZPage* new_page);
 };
 
 class ZPageTableIterator : public StackObj {
 private:
-  ZGranuleMapIterator<ZPage*> _iter;
-  ZPage*                      _prev;
+  ZGranuleMapIterator<ZPage*, false /* Parallel */> _iter;
+  ZPage*                                            _prev;
 
 public:
-  ZPageTableIterator(const ZPageTable* page_table);
+  ZPageTableIterator(const ZPageTable* table);
 
   bool next(ZPage** page);
 };
 
+class ZPageTableParallelIterator : public StackObj {
+  const ZPageTable* _table;
+  ZIndexDistributor _index_distributor;
+
+public:
+  ZPageTableParallelIterator(const ZPageTable* table);
+
+  template <typename Function>
+  void do_pages(Function function);
+};
+
+class ZGenerationPagesIterator : public StackObj {
+private:
+  ZPageTableIterator _iterator;
+  ZGenerationId      _generation_id;
+  ZPageAllocator*    _page_allocator;
+
+public:
+  ZGenerationPagesIterator(const ZPageTable* page_table, ZGenerationId id, ZPageAllocator* page_allocator);
+  ~ZGenerationPagesIterator();
+
+  bool next(ZPage** page);
+
+  template <typename Function>
+  void yield(Function function);
+};
+
+class ZGenerationPagesParallelIterator : public StackObj {
+private:
+  ZPageTableParallelIterator _iterator;
+  ZGenerationId              _generation_id;
+  ZPageAllocator*            _page_allocator;
+
+public:
+  ZGenerationPagesParallelIterator(const ZPageTable* page_table, ZGenerationId id, ZPageAllocator* page_allocator);
+  ~ZGenerationPagesParallelIterator();
+
+  template <typename Function>
+  void do_pages(Function function);
+};
+
 #endif // SHARE_GC_Z_ZPAGETABLE_HPP
diff --git a/src/hotspot/share/gc/z/zPageTable.inline.hpp b/src/hotspot/share/gc/z/zPageTable.inline.hpp
index d068958615d..16beeda89f2 100644
--- a/src/hotspot/share/gc/z/zPageTable.inline.hpp
+++ b/src/hotspot/share/gc/z/zPageTable.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,19 +28,30 @@
 
 #include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zGranuleMap.inline.hpp"
+#include "gc/z/zIndexDistributor.inline.hpp"
+#include "gc/z/zPage.inline.hpp"
+#include "gc/z/zPageAllocator.inline.hpp"
 
-inline ZPage* ZPageTable::get(uintptr_t addr) const {
-  assert(!ZAddress::is_null(addr), "Invalid address");
+inline ZPage* ZPageTable::get(zaddress addr) const {
+  assert(!is_null(addr), "Invalid address");
   return _map.get(ZAddress::offset(addr));
 }
 
-inline ZPageTableIterator::ZPageTableIterator(const ZPageTable* page_table) :
-    _iter(&page_table->_map),
-    _prev(NULL) {}
+inline ZPage* ZPageTable::get(volatile zpointer* p) const {
+  return get(to_zaddress((uintptr_t)p));
+}
+
+inline ZPage* ZPageTable::at(size_t index) const {
+  return _map.at(index);
+}
+
+inline ZPageTableIterator::ZPageTableIterator(const ZPageTable* table) :
+    _iter(&table->_map),
+    _prev(nullptr) {}
 
 inline bool ZPageTableIterator::next(ZPage** page) {
   for (ZPage* entry; _iter.next(&entry);) {
-    if (entry != NULL && entry != _prev) {
+    if (entry != nullptr && entry != _prev) {
       // Next page found
       *page = _prev = entry;
       return true;
@@ -51,4 +62,54 @@ inline bool ZPageTableIterator::next(ZPage** page) {
   return false;
 }
 
+inline ZPageTableParallelIterator::ZPageTableParallelIterator(const ZPageTable* table) :
+    _table(table),
+    _index_distributor(int(ZAddressOffsetMax >> ZGranuleSizeShift)) {}
+
+template <typename Function>
+inline void ZPageTableParallelIterator::do_pages(Function function) {
+  _index_distributor.do_indices([&](int index) {
+    ZPage* const page = _table->at(index);
+    if (page != nullptr) {
+      const size_t start_index = untype(page->start()) >> ZGranuleSizeShift;
+      if (size_t(index) == start_index) {
+        // Next page found
+        return function(page);
+      }
+    }
+    return true;
+  });
+}
+
+inline bool ZGenerationPagesIterator::next(ZPage** page) {
+  while (_iterator.next(page)) {
+    if ((*page)->generation_id() == _generation_id) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+template <typename Function>
+inline void ZGenerationPagesIterator::yield(Function function) {
+  _page_allocator->disable_safe_destroy();
+  _page_allocator->disable_safe_recycle();
+
+  function();
+
+  _page_allocator->enable_safe_recycle();
+  _page_allocator->enable_safe_destroy();
+}
+
+template <typename Function>
+inline void ZGenerationPagesParallelIterator::do_pages(Function function) {
+  _iterator.do_pages([&](ZPage* page) {
+    if (page->generation_id() == _generation_id) {
+      return function(page);
+    }
+    return true;
+  });
+}
+
 #endif // SHARE_GC_Z_ZPAGETABLE_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zPageType.hpp b/src/hotspot/share/gc/z/zPageType.hpp
new file mode 100644
index 00000000000..21b49305fae
--- /dev/null
+++ b/src/hotspot/share/gc/z/zPageType.hpp
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZPAGETYPE_HPP
+#define SHARE_GC_Z_ZPAGETYPE_HPP
+
+#include "utilities/globalDefinitions.hpp"
+
+enum class ZPageType : uint8_t {
+  small,
+  medium,
+  large
+};
+
+#endif // SHARE_GC_Z_ZPAGETYPE_HPP
diff --git a/src/hotspot/share/gc/z/zPhysicalMemory.cpp b/src/hotspot/share/gc/z/zPhysicalMemory.cpp
index eb889f4e4f2..f5e0a9c0994 100644
--- a/src/hotspot/share/gc/z/zPhysicalMemory.cpp
+++ b/src/hotspot/share/gc/z/zPhysicalMemory.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -73,11 +73,11 @@ size_t ZPhysicalMemory::size() const {
   return size;
 }
 
-void ZPhysicalMemory::insert_segment(int index, uintptr_t start, size_t size, bool committed) {
+void ZPhysicalMemory::insert_segment(int index, zoffset start, size_t size, bool committed) {
   _segments.insert_before(index, ZPhysicalMemorySegment(start, size, committed));
 }
 
-void ZPhysicalMemory::replace_segment(int index, uintptr_t start, size_t size, bool committed) {
+void ZPhysicalMemory::replace_segment(int index, zoffset start, size_t size, bool committed) {
   _segments.at_put(index, ZPhysicalMemorySegment(start, size, committed));
 }
 
@@ -108,7 +108,7 @@ void ZPhysicalMemory::add_segment(const ZPhysicalMemorySegment& segment) {
       if (is_mergable(_segments.at(current), segment)) {
         if (current + 1 < _segments.length() && is_mergable(segment, _segments.at(current + 1))) {
           // Merge with end of current segment and start of next segment
-          const size_t start = _segments.at(current).start();
+          const zoffset start = _segments.at(current).start();
           const size_t size = _segments.at(current).size() + segment.size() + _segments.at(current + 1).size();
           replace_segment(current, start, size, segment.is_committed());
           remove_segment(current + 1);
@@ -116,13 +116,13 @@ void ZPhysicalMemory::add_segment(const ZPhysicalMemorySegment& segment) {
         }
 
         // Merge with end of current segment
-        const size_t start = _segments.at(current).start();
+        const zoffset start = _segments.at(current).start();
         const size_t size = _segments.at(current).size() + segment.size();
         replace_segment(current, start, size, segment.is_committed());
         return;
       } else if (current + 1 < _segments.length() && is_mergable(segment, _segments.at(current + 1))) {
         // Merge with start of next segment
-        const size_t start = segment.start();
+        const zoffset start = segment.start();
         const size_t size = segment.size() + _segments.at(current + 1).size();
         replace_segment(current + 1, start, size, segment.is_committed());
         return;
@@ -136,7 +136,7 @@ void ZPhysicalMemory::add_segment(const ZPhysicalMemorySegment& segment) {
 
   if (_segments.length() > 0 && is_mergable(segment, _segments.at(0))) {
     // Merge with start of first segment
-    const size_t start = segment.start();
+    const zoffset start = segment.start();
     const size_t size = segment.size() + _segments.at(0).size();
     replace_segment(0, start, size, segment.is_committed());
     return;
@@ -234,7 +234,7 @@ ZPhysicalMemory ZPhysicalMemory::split_committed() {
 ZPhysicalMemoryManager::ZPhysicalMemoryManager(size_t max_capacity) :
     _backing(max_capacity) {
   // Make the whole range free
-  _manager.free(0, max_capacity);
+  _manager.free(zoffset(0), max_capacity);
 }
 
 bool ZPhysicalMemoryManager::is_initialized() const {
@@ -264,7 +264,7 @@ void ZPhysicalMemoryManager::try_enable_uncommit(size_t min_capacity, size_t max
 
   // Test if uncommit is supported by the operating system by committing
   // and then uncommitting a granule.
-  ZPhysicalMemory pmem(ZPhysicalMemorySegment(0, ZGranuleSize, false /* committed */));
+  ZPhysicalMemory pmem(ZPhysicalMemorySegment(zoffset(0), ZGranuleSize, false /* committed */));
   if (!commit(pmem) || !uncommit(pmem)) {
     log_info_p(gc, init)("Uncommit: Implicitly Disabled (Not supported by operating system)");
     FLAG_SET_ERGO(ZUncommit, false);
@@ -275,17 +275,16 @@ void ZPhysicalMemoryManager::try_enable_uncommit(size_t min_capacity, size_t max
   log_info_p(gc, init)("Uncommit Delay: " UINTX_FORMAT "s", ZUncommitDelay);
 }
 
-void ZPhysicalMemoryManager::nmt_commit(uintptr_t offset, size_t size) const {
-  // From an NMT point of view we treat the first heap view (marked0) as committed
-  const uintptr_t addr = ZAddress::marked0(offset);
-  MemTracker::record_virtual_memory_commit((void*)addr, size, CALLER_PC);
+void ZPhysicalMemoryManager::nmt_commit(zoffset offset, size_t size) const {
+  const zaddress addr = ZOffset::address(offset);
+  MemTracker::record_virtual_memory_commit((void*)untype(addr), size, CALLER_PC);
 }
 
-void ZPhysicalMemoryManager::nmt_uncommit(uintptr_t offset, size_t size) const {
+void ZPhysicalMemoryManager::nmt_uncommit(zoffset offset, size_t size) const {
   if (MemTracker::enabled()) {
-    const uintptr_t addr = ZAddress::marked0(offset);
+    const zaddress addr = ZOffset::address(offset);
     Tracker tracker(Tracker::uncommit);
-    tracker.record((address)addr, size);
+    tracker.record((address)untype(addr), size);
   }
 }
 
@@ -295,8 +294,8 @@ void ZPhysicalMemoryManager::alloc(ZPhysicalMemory& pmem, size_t size) {
   // Allocate segments
   while (size > 0) {
     size_t allocated = 0;
-    const uintptr_t start = _manager.alloc_low_address_at_most(size, &allocated);
-    assert(start != UINTPTR_MAX, "Allocation should never fail");
+    const zoffset start = _manager.alloc_low_address_at_most(size, &allocated);
+    assert(start != zoffset(UINTPTR_MAX), "Allocation should never fail");
     pmem.add_segment(ZPhysicalMemorySegment(start, allocated, false /* committed */));
     size -= allocated;
   }
@@ -352,12 +351,12 @@ bool ZPhysicalMemoryManager::uncommit(ZPhysicalMemory& pmem) {
   return true;
 }
 
-void ZPhysicalMemoryManager::pretouch_view(uintptr_t addr, size_t size) const {
+void ZPhysicalMemoryManager::pretouch_view(zaddress addr, size_t size) const {
   const size_t page_size = ZLargePages::is_explicit() ? ZGranuleSize : os::vm_page_size();
-  os::pretouch_memory((void*)addr, (void*)(addr + size), page_size);
+  os::pretouch_memory((void*)untype(addr), (void*)(untype(addr) + size), page_size);
 }
 
-void ZPhysicalMemoryManager::map_view(uintptr_t addr, const ZPhysicalMemory& pmem) const {
+void ZPhysicalMemoryManager::map_view(zaddress_unsafe addr, const ZPhysicalMemory& pmem) const {
   size_t size = 0;
 
   // Map segments
@@ -376,60 +375,27 @@ void ZPhysicalMemoryManager::map_view(uintptr_t addr, const ZPhysicalMemory& pme
   }
 }
 
-void ZPhysicalMemoryManager::unmap_view(uintptr_t addr, size_t size) const {
+void ZPhysicalMemoryManager::unmap_view(zaddress_unsafe addr, size_t size) const {
   _backing.unmap(addr, size);
 }
 
-void ZPhysicalMemoryManager::pretouch(uintptr_t offset, size_t size) const {
-  if (ZVerifyViews) {
-    // Pre-touch good view
-    pretouch_view(ZAddress::good(offset), size);
-  } else {
-    // Pre-touch all views
-    pretouch_view(ZAddress::marked0(offset), size);
-    pretouch_view(ZAddress::marked1(offset), size);
-    pretouch_view(ZAddress::remapped(offset), size);
-  }
+void ZPhysicalMemoryManager::pretouch(zoffset offset, size_t size) const {
+  // Pre-touch all views
+  pretouch_view(ZOffset::address(offset), size);
 }
 
-void ZPhysicalMemoryManager::map(uintptr_t offset, const ZPhysicalMemory& pmem) const {
+void ZPhysicalMemoryManager::map(zoffset offset, const ZPhysicalMemory& pmem) const {
   const size_t size = pmem.size();
 
-  if (ZVerifyViews) {
-    // Map good view
-    map_view(ZAddress::good(offset), pmem);
-  } else {
-    // Map all views
-    map_view(ZAddress::marked0(offset), pmem);
-    map_view(ZAddress::marked1(offset), pmem);
-    map_view(ZAddress::remapped(offset), pmem);
-  }
+  // Map all views
+  map_view(ZOffset::address_unsafe(offset), pmem);
 
   nmt_commit(offset, size);
 }
 
-void ZPhysicalMemoryManager::unmap(uintptr_t offset, size_t size) const {
+void ZPhysicalMemoryManager::unmap(zoffset offset, size_t size) const {
   nmt_uncommit(offset, size);
 
-  if (ZVerifyViews) {
-    // Unmap good view
-    unmap_view(ZAddress::good(offset), size);
-  } else {
-    // Unmap all views
-    unmap_view(ZAddress::marked0(offset), size);
-    unmap_view(ZAddress::marked1(offset), size);
-    unmap_view(ZAddress::remapped(offset), size);
-  }
-}
-
-void ZPhysicalMemoryManager::debug_map(uintptr_t offset, const ZPhysicalMemory& pmem) const {
-  // Map good view
-  assert(ZVerifyViews, "Should be enabled");
-  map_view(ZAddress::good(offset), pmem);
-}
-
-void ZPhysicalMemoryManager::debug_unmap(uintptr_t offset, size_t size) const {
-  // Unmap good view
-  assert(ZVerifyViews, "Should be enabled");
-  unmap_view(ZAddress::good(offset), size);
+  // Unmap all views
+  unmap_view(ZOffset::address_unsafe(offset), size);
 }
diff --git a/src/hotspot/share/gc/z/zPhysicalMemory.hpp b/src/hotspot/share/gc/z/zPhysicalMemory.hpp
index 8332eca32e1..0cf77dad739 100644
--- a/src/hotspot/share/gc/z/zPhysicalMemory.hpp
+++ b/src/hotspot/share/gc/z/zPhysicalMemory.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
 #ifndef SHARE_GC_Z_ZPHYSICALMEMORY_HPP
 #define SHARE_GC_Z_ZPHYSICALMEMORY_HPP
 
+#include "gc/z/zAddress.hpp"
 #include "gc/z/zArray.hpp"
 #include "gc/z/zMemory.hpp"
 #include "memory/allocation.hpp"
@@ -31,16 +32,16 @@
 
 class ZPhysicalMemorySegment : public CHeapObj<mtGC> {
 private:
-  uintptr_t _start;
-  uintptr_t _end;
-  bool      _committed;
+  zoffset _start;
+  zoffset _end;
+  bool    _committed;
 
 public:
   ZPhysicalMemorySegment();
-  ZPhysicalMemorySegment(uintptr_t start, size_t size, bool committed);
+  ZPhysicalMemorySegment(zoffset start, size_t size, bool committed);
 
-  uintptr_t start() const;
-  uintptr_t end() const;
+  zoffset start() const;
+  zoffset end() const;
   size_t size() const;
 
   bool is_committed() const;
@@ -51,8 +52,8 @@ class ZPhysicalMemory {
 private:
   ZArray<ZPhysicalMemorySegment> _segments;
 
-  void insert_segment(int index, uintptr_t start, size_t size, bool committed);
-  void replace_segment(int index, uintptr_t start, size_t size, bool committed);
+  void insert_segment(int index, zoffset start, size_t size, bool committed);
+  void replace_segment(int index, zoffset start, size_t size, bool committed);
   void remove_segment(int index);
 
 public:
@@ -83,12 +84,12 @@ private:
   ZPhysicalMemoryBacking _backing;
   ZMemoryManager         _manager;
 
-  void nmt_commit(uintptr_t offset, size_t size) const;
-  void nmt_uncommit(uintptr_t offset, size_t size) const;
+  void nmt_commit(zoffset offset, size_t size) const;
+  void nmt_uncommit(zoffset offset, size_t size) const;
 
-  void pretouch_view(uintptr_t addr, size_t size) const;
-  void map_view(uintptr_t addr, const ZPhysicalMemory& pmem) const;
-  void unmap_view(uintptr_t addr, size_t size) const;
+  void pretouch_view(zaddress addr, size_t size) const;
+  void map_view(zaddress_unsafe addr, const ZPhysicalMemory& pmem) const;
+  void unmap_view(zaddress_unsafe addr, size_t size) const;
 
 public:
   ZPhysicalMemoryManager(size_t max_capacity);
@@ -104,13 +105,10 @@ public:
   bool commit(ZPhysicalMemory& pmem);
   bool uncommit(ZPhysicalMemory& pmem);
 
-  void pretouch(uintptr_t offset, size_t size) const;
+  void pretouch(zoffset offset, size_t size) const;
 
-  void map(uintptr_t offset, const ZPhysicalMemory& pmem) const;
-  void unmap(uintptr_t offset, size_t size) const;
-
-  void debug_map(uintptr_t offset, const ZPhysicalMemory& pmem) const;
-  void debug_unmap(uintptr_t offset, size_t size) const;
+  void map(zoffset offset, const ZPhysicalMemory& pmem) const;
+  void unmap(zoffset offset, size_t size) const;
 };
 
 #endif // SHARE_GC_Z_ZPHYSICALMEMORY_HPP
diff --git a/src/hotspot/share/gc/z/zPhysicalMemory.inline.hpp b/src/hotspot/share/gc/z/zPhysicalMemory.inline.hpp
index b2411379935..c6c434f57b6 100644
--- a/src/hotspot/share/gc/z/zPhysicalMemory.inline.hpp
+++ b/src/hotspot/share/gc/z/zPhysicalMemory.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,20 +30,20 @@
 #include "utilities/debug.hpp"
 
 inline ZPhysicalMemorySegment::ZPhysicalMemorySegment() :
-    _start(UINTPTR_MAX),
-    _end(UINTPTR_MAX),
+    _start(zoffset(UINTPTR_MAX)),
+    _end(zoffset(UINTPTR_MAX)),
     _committed(false) {}
 
-inline ZPhysicalMemorySegment::ZPhysicalMemorySegment(uintptr_t start, size_t size, bool committed) :
+inline ZPhysicalMemorySegment::ZPhysicalMemorySegment(zoffset start, size_t size, bool committed) :
     _start(start),
     _end(start + size),
     _committed(committed) {}
 
-inline uintptr_t ZPhysicalMemorySegment::start() const {
+inline zoffset ZPhysicalMemorySegment::start() const {
   return _start;
 }
 
-inline uintptr_t ZPhysicalMemorySegment::end() const {
+inline zoffset ZPhysicalMemorySegment::end() const {
   return _end;
 }
 
diff --git a/src/hotspot/share/gc/z/zReferenceProcessor.cpp b/src/hotspot/share/gc/z/zReferenceProcessor.cpp
index 042323c8777..6da4cbdf4da 100644
--- a/src/hotspot/share/gc/z/zReferenceProcessor.cpp
+++ b/src/hotspot/share/gc/z/zReferenceProcessor.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,9 @@
 #include "classfile/javaClasses.inline.hpp"
 #include "gc/shared/referencePolicy.hpp"
 #include "gc/shared/referenceProcessorStats.hpp"
+#include "gc/shared/suspendibleThreadSet.hpp"
+#include "gc/z/zCollectedHeap.hpp"
+#include "gc/z/zDriver.hpp"
 #include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zReferenceProcessor.hpp"
 #include "gc/z/zStat.hpp"
@@ -32,15 +35,16 @@
 #include "gc/z/zTracer.inline.hpp"
 #include "gc/z/zValue.inline.hpp"
 #include "memory/universe.hpp"
+#include "oops/access.inline.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/os.hpp"
 
-static const ZStatSubPhase ZSubPhaseConcurrentReferencesProcess("Concurrent References Process");
-static const ZStatSubPhase ZSubPhaseConcurrentReferencesEnqueue("Concurrent References Enqueue");
+static const ZStatSubPhase ZSubPhaseConcurrentReferencesProcess("Concurrent References Process", ZGenerationId::old);
+static const ZStatSubPhase ZSubPhaseConcurrentReferencesEnqueue("Concurrent References Enqueue", ZGenerationId::old);
 
-static ReferenceType reference_type(oop reference) {
-  return InstanceKlass::cast(reference->klass())->reference_type();
+static ReferenceType reference_type(zaddress reference) {
+  return InstanceKlass::cast(to_oop(reference)->klass())->reference_type();
 }
 
 static const char* reference_type_name(ReferenceType type) {
@@ -63,56 +67,58 @@ static const char* reference_type_name(ReferenceType type) {
   }
 }
 
-static volatile oop* reference_referent_addr(oop reference) {
-  return (volatile oop*)java_lang_ref_Reference::referent_addr_raw(reference);
+static volatile zpointer* reference_referent_addr(zaddress reference) {
+  return (volatile zpointer*)java_lang_ref_Reference::referent_addr_raw(to_oop(reference));
 }
 
-static oop reference_referent(oop reference) {
-  return Atomic::load(reference_referent_addr(reference));
+static zpointer reference_referent(zaddress reference) {
+  return ZBarrier::load_atomic(reference_referent_addr(reference));
 }
 
-static void reference_clear_referent(oop reference) {
-  java_lang_ref_Reference::clear_referent_raw(reference);
+static zaddress reference_discovered(zaddress reference) {
+  return to_zaddress(java_lang_ref_Reference::discovered(to_oop(reference)));
 }
 
-static oop* reference_discovered_addr(oop reference) {
-  return (oop*)java_lang_ref_Reference::discovered_addr_raw(reference);
+static void reference_set_discovered(zaddress reference, zaddress discovered) {
+  java_lang_ref_Reference::set_discovered(to_oop(reference), to_oop(discovered));
 }
 
-static oop reference_discovered(oop reference) {
-  return *reference_discovered_addr(reference);
+static zaddress reference_next(zaddress reference) {
+  return to_zaddress(java_lang_ref_Reference::next(to_oop(reference)));
 }
 
-static void reference_set_discovered(oop reference, oop discovered) {
-  java_lang_ref_Reference::set_discovered_raw(reference, discovered);
-}
-
-static oop* reference_next_addr(oop reference) {
-  return (oop*)java_lang_ref_Reference::next_addr_raw(reference);
-}
-
-static oop reference_next(oop reference) {
-  return *reference_next_addr(reference);
-}
-
-static void reference_set_next(oop reference, oop next) {
-  java_lang_ref_Reference::set_next_raw(reference, next);
+static void reference_set_next(zaddress reference, zaddress next) {
+  java_lang_ref_Reference::set_next(to_oop(reference), to_oop(next));
 }
 
 static void soft_reference_update_clock() {
+  SuspendibleThreadSetJoiner sts_joiner;
   const jlong now = os::javaTimeNanos() / NANOSECS_PER_MILLISEC;
   java_lang_ref_SoftReference::set_clock(now);
 }
 
+static void list_append(zaddress& head, zaddress& tail, zaddress reference) {
+  if (is_null(head)) {
+    // First append - set up the head
+    head = reference;
+  } else {
+    // Not first append, link tail
+    reference_set_discovered(tail, reference);
+  }
+
+  // Always set tail
+  tail = reference;
+}
+
 ZReferenceProcessor::ZReferenceProcessor(ZWorkers* workers) :
     _workers(workers),
-    _soft_reference_policy(NULL),
+    _soft_reference_policy(nullptr),
     _encountered_count(),
     _discovered_count(),
     _enqueued_count(),
-    _discovered_list(NULL),
-    _pending_list(NULL),
-    _pending_list_tail(_pending_list.addr()) {}
+    _discovered_list(zaddress::null),
+    _pending_list(zaddress::null),
+    _pending_list_tail(zaddress::null) {}
 
 void ZReferenceProcessor::set_soft_reference_policy(bool clear) {
   static AlwaysClearPolicy always_clear_policy;
@@ -128,23 +134,27 @@ void ZReferenceProcessor::set_soft_reference_policy(bool clear) {
   _soft_reference_policy->setup();
 }
 
-bool ZReferenceProcessor::is_inactive(oop reference, oop referent, ReferenceType type) const {
+bool ZReferenceProcessor::is_inactive(zaddress reference, oop referent, ReferenceType type) const {
   if (type == REF_FINAL) {
     // A FinalReference is inactive if its next field is non-null. An application can't
     // call enqueue() or clear() on a FinalReference.
-    return reference_next(reference) != NULL;
+    return !is_null(reference_next(reference));
   } else {
+    // Verification
+    (void)to_zaddress(referent);
+
     // A non-FinalReference is inactive if the referent is null. The referent can only
     // be null if the application called Reference.enqueue() or Reference.clear().
-    return referent == NULL;
+    return referent == nullptr;
   }
 }
 
 bool ZReferenceProcessor::is_strongly_live(oop referent) const {
-  return ZHeap::heap()->is_object_strongly_live(ZOop::to_address(referent));
+  const zaddress addr = to_zaddress(referent);
+  return ZHeap::heap()->is_young(addr) || ZHeap::heap()->is_object_strongly_live(to_zaddress(referent));
 }
 
-bool ZReferenceProcessor::is_softly_live(oop reference, ReferenceType type) const {
+bool ZReferenceProcessor::is_softly_live(zaddress reference, ReferenceType type) const {
   if (type != REF_SOFT) {
     // Not a SoftReference
     return false;
@@ -153,18 +163,22 @@ bool ZReferenceProcessor::is_softly_live(oop reference, ReferenceType type) cons
   // Ask SoftReference policy
   const jlong clock = java_lang_ref_SoftReference::clock();
   assert(clock != 0, "Clock not initialized");
-  assert(_soft_reference_policy != NULL, "Policy not initialized");
-  return !_soft_reference_policy->should_clear_reference(reference, clock);
+  assert(_soft_reference_policy != nullptr, "Policy not initialized");
+  return !_soft_reference_policy->should_clear_reference(to_oop(reference), clock);
 }
 
-bool ZReferenceProcessor::should_discover(oop reference, ReferenceType type) const {
-  volatile oop* const referent_addr = reference_referent_addr(reference);
-  const oop referent = ZBarrier::weak_load_barrier_on_oop_field(referent_addr);
+bool ZReferenceProcessor::should_discover(zaddress reference, ReferenceType type) const {
+  volatile zpointer* const referent_addr = reference_referent_addr(reference);
+  const oop referent = to_oop(ZBarrier::load_barrier_on_oop_field(referent_addr));
 
   if (is_inactive(reference, referent, type)) {
     return false;
   }
 
+  if (ZHeap::heap()->is_young(reference)) {
+    return false;
+  }
+
   if (is_strongly_live(referent)) {
     return false;
   }
@@ -182,49 +196,43 @@ bool ZReferenceProcessor::should_discover(oop reference, ReferenceType type) con
   return true;
 }
 
-bool ZReferenceProcessor::should_drop(oop reference, ReferenceType type) const {
-  const oop referent = reference_referent(reference);
-  if (referent == NULL) {
-    // Reference has been cleared, by a call to Reference.enqueue()
-    // or Reference.clear() from the application, which means we
-    // should drop the reference.
-    return true;
+bool ZReferenceProcessor::try_make_inactive(zaddress reference, ReferenceType type) const {
+  const zpointer referent = reference_referent(reference);
+
+  if (is_null_any(referent)) {
+    // Reference has already been cleared, by a call to Reference.enqueue()
+    // or Reference.clear() from the application, which means it's already
+    // inactive and we should drop the reference.
+    return false;
   }
 
-  // Check if the referent is still alive, in which case we should
-  // drop the reference.
-  if (type == REF_PHANTOM) {
-    return ZBarrier::is_alive_barrier_on_phantom_oop(referent);
+  volatile zpointer* const referent_addr = reference_referent_addr(reference);
+
+  // Cleaning the referent will fail if the object it points to is
+  // still alive, in which case we should drop the reference.
+  if (type == REF_SOFT || type == REF_WEAK) {
+    return ZBarrier::clean_barrier_on_weak_oop_field(referent_addr);
+  } else if (type == REF_PHANTOM) {
+    return ZBarrier::clean_barrier_on_phantom_oop_field(referent_addr);
+  } else if (type == REF_FINAL) {
+    if (ZBarrier::clean_barrier_on_final_oop_field(referent_addr)) {
+      // The referent in a FinalReference will not be cleared, instead it is
+      // made inactive by self-looping the next field. An application can't
+      // call FinalReference.enqueue(), so there is no race to worry about
+      // when setting the next field.
+      assert(is_null(reference_next(reference)), "Already inactive");
+      reference_set_next(reference, reference);
+      return true;
+    }
   } else {
-    return ZBarrier::is_alive_barrier_on_weak_oop(referent);
+    fatal("Invalid referent type %d", type);
   }
+
+  return false;
 }
 
-void ZReferenceProcessor::keep_alive(oop reference, ReferenceType type) const {
-  volatile oop* const p = reference_referent_addr(reference);
-  if (type == REF_PHANTOM) {
-    ZBarrier::keep_alive_barrier_on_phantom_oop_field(p);
-  } else {
-    ZBarrier::keep_alive_barrier_on_weak_oop_field(p);
-  }
-}
-
-void ZReferenceProcessor::make_inactive(oop reference, ReferenceType type) const {
-  if (type == REF_FINAL) {
-    // Don't clear referent. It is needed by the Finalizer thread to make the call
-    // to finalize(). A FinalReference is instead made inactive by self-looping the
-    // next field. An application can't call FinalReference.enqueue(), so there is
-    // no race to worry about when setting the next field.
-    assert(reference_next(reference) == NULL, "Already inactive");
-    reference_set_next(reference, reference);
-  } else {
-    // Clear referent
-    reference_clear_referent(reference);
-  }
-}
-
-void ZReferenceProcessor::discover(oop reference, ReferenceType type) {
-  log_trace(gc, ref)("Discovered Reference: " PTR_FORMAT " (%s)", p2i(reference), reference_type_name(type));
+void ZReferenceProcessor::discover(zaddress reference, ReferenceType type) {
+  log_trace(gc, ref)("Discovered Reference: " PTR_FORMAT " (%s)", untype(reference), reference_type_name(type));
 
   // Update statistics
   _discovered_count.get()[type]++;
@@ -233,24 +241,27 @@ void ZReferenceProcessor::discover(oop reference, ReferenceType type) {
     // Mark referent (and its reachable subgraph) finalizable. This avoids
     // the problem of later having to mark those objects if the referent is
     // still final reachable during processing.
-    volatile oop* const referent_addr = reference_referent_addr(reference);
-    ZBarrier::mark_barrier_on_oop_field(referent_addr, true /* finalizable */);
+    volatile zpointer* const referent_addr = reference_referent_addr(reference);
+    ZBarrier::mark_barrier_on_old_oop_field(referent_addr, true /* finalizable */);
   }
 
   // Add reference to discovered list
-  assert(reference_discovered(reference) == NULL, "Already discovered");
-  oop* const list = _discovered_list.addr();
+  assert(ZHeap::heap()->is_old(reference), "Must be old");
+  assert(is_null(reference_discovered(reference)), "Already discovered");
+  zaddress* const list = _discovered_list.addr();
   reference_set_discovered(reference, *list);
   *list = reference;
 }
 
-bool ZReferenceProcessor::discover_reference(oop reference, ReferenceType type) {
+bool ZReferenceProcessor::discover_reference(oop reference_obj, ReferenceType type) {
   if (!RegisterReferences) {
     // Reference processing disabled
     return false;
   }
 
-  log_trace(gc, ref)("Encountered Reference: " PTR_FORMAT " (%s)", p2i(reference), reference_type_name(type));
+  log_trace(gc, ref)("Encountered Reference: " PTR_FORMAT " (%s)", p2i(reference_obj), reference_type_name(type));
+
+  const zaddress reference = to_zaddress(reference_obj);
 
   // Update statistics
   _encountered_count.get()[type]++;
@@ -266,77 +277,81 @@ bool ZReferenceProcessor::discover_reference(oop reference, ReferenceType type)
   return true;
 }
 
-oop ZReferenceProcessor::drop(oop reference, ReferenceType type) {
-  log_trace(gc, ref)("Dropped Reference: " PTR_FORMAT " (%s)", p2i(reference), reference_type_name(type));
+void ZReferenceProcessor::process_worker_discovered_list(zaddress discovered_list) {
+  zaddress keep_head = zaddress::null;
+  zaddress keep_tail = zaddress::null;
 
-  // Keep referent alive
-  keep_alive(reference, type);
+  // Iterate over the discovered list and unlink them as we go, potentially
+  // appending them to the keep list
+  for (zaddress reference = discovered_list; !is_null(reference); ) {
+    assert(ZHeap::heap()->is_old(reference), "Must be old");
 
-  // Unlink and return next in list
-  const oop next = reference_discovered(reference);
-  reference_set_discovered(reference, NULL);
-  return next;
-}
-
-oop* ZReferenceProcessor::keep(oop reference, ReferenceType type) {
-  log_trace(gc, ref)("Enqueued Reference: " PTR_FORMAT " (%s)", p2i(reference), reference_type_name(type));
-
-  // Update statistics
-  _enqueued_count.get()[type]++;
-
-  // Make reference inactive
-  make_inactive(reference, type);
-
-  // Return next in list
-  return reference_discovered_addr(reference);
-}
-
-void ZReferenceProcessor::work() {
-  // Process discovered references
-  oop* const list = _discovered_list.addr();
-  oop* p = list;
-
-  while (*p != NULL) {
-    const oop reference = *p;
     const ReferenceType type = reference_type(reference);
+    const zaddress next = reference_discovered(reference);
+    reference_set_discovered(reference, zaddress::null);
 
-    if (should_drop(reference, type)) {
-      *p = drop(reference, type);
+    if (try_make_inactive(reference, type)) {
+      // Keep reference
+      log_trace(gc, ref)("Enqueued Reference: " PTR_FORMAT " (%s)", untype(reference), reference_type_name(type));
+
+      // Update statistics
+      _enqueued_count.get()[type]++;
+
+      list_append(keep_head, keep_tail, reference);
     } else {
-      p = keep(reference, type);
+      // Drop reference
+      log_trace(gc, ref)("Dropped Reference: " PTR_FORMAT " (%s)", untype(reference), reference_type_name(type));
     }
+
+    reference = next;
+    SuspendibleThreadSet::yield();
   }
 
   // Prepend discovered references to internal pending list
-  if (*list != NULL) {
-    *p = Atomic::xchg(_pending_list.addr(), *list);
-    if (*p == NULL) {
-      // First to prepend to list, record tail
-      _pending_list_tail = p;
-    }
 
-    // Clear discovered list
-    *list = NULL;
+  // Anything kept on the list?
+  if (!is_null(keep_head)) {
+    const zaddress old_pending_list = Atomic::xchg(_pending_list.addr(), keep_head);
+
+    // Concatenate the old list
+    reference_set_discovered(keep_tail, old_pending_list);
+
+    if (is_null(old_pending_list)) {
+      // Old list was empty. First to prepend to list, record tail
+      _pending_list_tail = keep_tail;
+    } else {
+      assert(ZHeap::heap()->is_old(old_pending_list), "Must be old");
+    }
   }
 }
 
-bool ZReferenceProcessor::is_empty() const {
-  ZPerWorkerConstIterator<oop> iter(&_discovered_list);
-  for (const oop* list; iter.next(&list);) {
-    if (*list != NULL) {
-      return false;
+void ZReferenceProcessor::work() {
+  SuspendibleThreadSetJoiner sts_joiner;
+
+  ZPerWorkerIterator<zaddress> iter(&_discovered_list);
+  for (zaddress* start; iter.next(&start);) {
+    const zaddress discovered_list = Atomic::xchg(start, zaddress::null);
+
+    if (discovered_list != zaddress::null) {
+      // Process discovered references
+      process_worker_discovered_list(discovered_list);
     }
   }
+}
 
-  if (_pending_list.get() != NULL) {
-    return false;
+void ZReferenceProcessor::verify_empty() const {
+#ifdef ASSERT
+  ZPerWorkerConstIterator<zaddress> iter(&_discovered_list);
+  for (const zaddress* list; iter.next(&list);) {
+    assert(is_null(*list), "Discovered list not empty");
   }
 
-  return true;
+  assert(is_null(_pending_list.get()), "Pending list not empty");
+#endif
 }
 
 void ZReferenceProcessor::reset_statistics() {
-  assert(is_empty(), "Should be empty");
+  verify_empty();
 
   // Reset encountered
   ZPerWorkerIterator<Counters> iter_encountered(&_encountered_count);
@@ -403,7 +418,7 @@ void ZReferenceProcessor::collect_statistics() {
                                       discovered[REF_WEAK],
                                       discovered[REF_FINAL],
                                       discovered[REF_PHANTOM]);
-  ZTracer::tracer()->report_gc_reference_stats(stats);
+  ZDriver::major()->jfr_tracer()->report_gc_reference_stats(stats);
 }
 
 class ZReferenceProcessorTask : public ZTask {
@@ -421,7 +436,7 @@ public:
 };
 
 void ZReferenceProcessor::process_references() {
-  ZStatTimer timer(ZSubPhaseConcurrentReferencesProcess);
+  ZStatTimerOld timer(ZSubPhaseConcurrentReferencesProcess);
 
   // Process discovered lists
   ZReferenceProcessorTask task(this);
@@ -434,26 +449,61 @@ void ZReferenceProcessor::process_references() {
   collect_statistics();
 }
 
-void ZReferenceProcessor::enqueue_references() {
-  ZStatTimer timer(ZSubPhaseConcurrentReferencesEnqueue);
+void ZReferenceProcessor::verify_pending_references() {
+#ifdef ASSERT
+  SuspendibleThreadSetJoiner sts_joiner;
 
-  if (_pending_list.get() == NULL) {
+  assert(!is_null(_pending_list.get()), "Should not contain colored null");
+
+  for (zaddress current = _pending_list.get();
+       !is_null(current);
+       current = reference_discovered(current))
+  {
+    volatile zpointer* const referent_addr = reference_referent_addr(current);
+    const oop referent = to_oop(ZBarrier::load_barrier_on_oop_field(referent_addr));
+    const ReferenceType type = reference_type(current);
+    assert(ZReferenceProcessor::is_inactive(current, referent, type), "invariant");
+    if (type == REF_FINAL) {
+      assert(ZPointer::is_marked_any_old(ZBarrier::load_atomic(referent_addr)), "invariant");
+    }
+
+    SuspendibleThreadSet::yield();
+  }
+#endif
+}
+
+zaddress ZReferenceProcessor::swap_pending_list(zaddress pending_list) {
+  const oop pending_list_oop = to_oop(pending_list);
+  const oop prev = Universe::swap_reference_pending_list(pending_list_oop);
+  return to_zaddress(prev);
+}
+
+void ZReferenceProcessor::enqueue_references() {
+  ZStatTimerOld timer(ZSubPhaseConcurrentReferencesEnqueue);
+
+  if (is_null(_pending_list.get())) {
     // Nothing to enqueue
     return;
   }
 
+  // Verify references on internal pending list
+  verify_pending_references();
+
   {
     // Heap_lock protects external pending list
     MonitorLocker ml(Heap_lock);
+    SuspendibleThreadSetJoiner sts_joiner;
 
-    // Prepend internal pending list to external pending list
-    *_pending_list_tail = Universe::swap_reference_pending_list(_pending_list.get());
+    const zaddress prev_list = swap_pending_list(_pending_list.get());
+
+    // Link together new and old list
+    reference_set_discovered(_pending_list_tail, prev_list);
 
     // Notify ReferenceHandler thread
     ml.notify_all();
   }
 
   // Reset internal pending list
-  _pending_list.set(NULL);
-  _pending_list_tail = _pending_list.addr();
+  _pending_list.set(zaddress::null);
+  _pending_list_tail = zaddress::null;
 }
diff --git a/src/hotspot/share/gc/z/zReferenceProcessor.hpp b/src/hotspot/share/gc/z/zReferenceProcessor.hpp
index aae9bfeb8a1..d39cc8634cd 100644
--- a/src/hotspot/share/gc/z/zReferenceProcessor.hpp
+++ b/src/hotspot/share/gc/z/zReferenceProcessor.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,8 +25,10 @@
 #define SHARE_GC_Z_ZREFERENCEPROCESSOR_HPP
 
 #include "gc/shared/referenceDiscoverer.hpp"
+#include "gc/z/zAddress.hpp"
 #include "gc/z/zValue.hpp"
 
+class ConcurrentGCTimer;
 class ReferencePolicy;
 class ZWorkers;
 
@@ -42,29 +44,27 @@ private:
   ZPerWorker<Counters> _encountered_count;
   ZPerWorker<Counters> _discovered_count;
   ZPerWorker<Counters> _enqueued_count;
-  ZPerWorker<oop>      _discovered_list;
-  ZContended<oop>      _pending_list;
-  oop*                 _pending_list_tail;
+  ZPerWorker<zaddress> _discovered_list;
+  ZContended<zaddress> _pending_list;
+  zaddress             _pending_list_tail;
 
-  bool is_inactive(oop reference, oop referent, ReferenceType type) const;
+  bool is_inactive(zaddress reference, oop referent, ReferenceType type) const;
   bool is_strongly_live(oop referent) const;
-  bool is_softly_live(oop reference, ReferenceType type) const;
+  bool is_softly_live(zaddress reference, ReferenceType type) const;
 
-  bool should_discover(oop reference, ReferenceType type) const;
-  bool should_drop(oop reference, ReferenceType type) const;
-  void keep_alive(oop reference, ReferenceType type) const;
-  void make_inactive(oop reference, ReferenceType type) const;
+  bool should_discover(zaddress reference, ReferenceType type) const;
+  bool try_make_inactive(zaddress reference, ReferenceType type) const;
 
-  void discover(oop reference, ReferenceType type);
+  void discover(zaddress reference, ReferenceType type);
 
-  oop drop(oop reference, ReferenceType type);
-  oop* keep(oop reference, ReferenceType type);
-
-  bool is_empty() const;
+  void verify_empty() const;
 
+  void process_worker_discovered_list(zaddress discovered_list);
   void work();
   void collect_statistics();
 
+  zaddress swap_pending_list(zaddress pending_list);
+
 public:
   ZReferenceProcessor(ZWorkers* workers);
 
@@ -74,6 +74,8 @@ public:
   virtual bool discover_reference(oop reference, ReferenceType type);
   void process_references();
   void enqueue_references();
+
+  void verify_pending_references();
 };
 
 #endif // SHARE_GC_Z_ZREFERENCEPROCESSOR_HPP
diff --git a/src/hotspot/share/gc/z/zRelocate.cpp b/src/hotspot/share/gc/z/zRelocate.cpp
index bc5147544a9..0ec474f0873 100644
--- a/src/hotspot/share/gc/z/zRelocate.cpp
+++ b/src/hotspot/share/gc/z/zRelocate.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,147 +23,435 @@
 
 #include "precompiled.hpp"
 #include "gc/shared/gc_globals.hpp"
+#include "gc/shared/suspendibleThreadSet.hpp"
 #include "gc/z/zAbort.inline.hpp"
 #include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zAllocator.inline.hpp"
 #include "gc/z/zBarrier.inline.hpp"
+#include "gc/z/zCollectedHeap.hpp"
 #include "gc/z/zForwarding.inline.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zHeap.inline.hpp"
+#include "gc/z/zIndexDistributor.inline.hpp"
+#include "gc/z/zIterator.inline.hpp"
 #include "gc/z/zPage.inline.hpp"
+#include "gc/z/zPageAge.hpp"
 #include "gc/z/zRelocate.hpp"
 #include "gc/z/zRelocationSet.inline.hpp"
+#include "gc/z/zRootsIterator.hpp"
+#include "gc/z/zStackWatermark.hpp"
 #include "gc/z/zStat.hpp"
 #include "gc/z/zTask.hpp"
-#include "gc/z/zThread.inline.hpp"
+#include "gc/z/zUncoloredRoot.inline.hpp"
+#include "gc/z/zVerify.hpp"
 #include "gc/z/zWorkers.hpp"
 #include "prims/jvmtiTagMap.hpp"
 #include "runtime/atomic.hpp"
 #include "utilities/debug.hpp"
 
-ZRelocate::ZRelocate(ZWorkers* workers) :
-    _workers(workers) {}
+static const ZStatCriticalPhase ZCriticalPhaseRelocationStall("Relocation Stall");
+static const ZStatSubPhase ZSubPhaseConcurrentRelocateRememberedSetFlipPromotedYoung("Concurrent Relocate Remset FP", ZGenerationId::young);
 
-static uintptr_t forwarding_index(ZForwarding* forwarding, uintptr_t from_addr) {
-  const uintptr_t from_offset = ZAddress::offset(from_addr);
+static uintptr_t forwarding_index(ZForwarding* forwarding, zoffset from_offset) {
   return (from_offset - forwarding->start()) >> forwarding->object_alignment_shift();
 }
 
-static uintptr_t forwarding_find(ZForwarding* forwarding, uintptr_t from_addr, ZForwardingCursor* cursor) {
-  const uintptr_t from_index = forwarding_index(forwarding, from_addr);
+static zaddress forwarding_find(ZForwarding* forwarding, zoffset from_offset, ZForwardingCursor* cursor) {
+  const uintptr_t from_index = forwarding_index(forwarding, from_offset);
   const ZForwardingEntry entry = forwarding->find(from_index, cursor);
-  return entry.populated() ? ZAddress::good(entry.to_offset()) : 0;
+  return entry.populated() ? ZOffset::address(to_zoffset(entry.to_offset())) : zaddress::null;
 }
 
-static uintptr_t forwarding_insert(ZForwarding* forwarding, uintptr_t from_addr, uintptr_t to_addr, ZForwardingCursor* cursor) {
-  const uintptr_t from_index = forwarding_index(forwarding, from_addr);
-  const uintptr_t to_offset = ZAddress::offset(to_addr);
-  const uintptr_t to_offset_final = forwarding->insert(from_index, to_offset, cursor);
-  return ZAddress::good(to_offset_final);
+static zaddress forwarding_find(ZForwarding* forwarding, zaddress_unsafe from_addr, ZForwardingCursor* cursor) {
+  return forwarding_find(forwarding, ZAddress::offset(from_addr), cursor);
 }
 
-static uintptr_t relocate_object_inner(ZForwarding* forwarding, uintptr_t from_addr, ZForwardingCursor* cursor) {
+static zaddress forwarding_find(ZForwarding* forwarding, zaddress from_addr, ZForwardingCursor* cursor) {
+  return forwarding_find(forwarding, ZAddress::offset(from_addr), cursor);
+}
+
+static zaddress forwarding_insert(ZForwarding* forwarding, zoffset from_offset, zaddress to_addr, ZForwardingCursor* cursor) {
+  const uintptr_t from_index = forwarding_index(forwarding, from_offset);
+  const zoffset to_offset = ZAddress::offset(to_addr);
+  const zoffset to_offset_final = forwarding->insert(from_index, to_offset, cursor);
+  return ZOffset::address(to_offset_final);
+}
+
+static zaddress forwarding_insert(ZForwarding* forwarding, zaddress from_addr, zaddress to_addr, ZForwardingCursor* cursor) {
+  return forwarding_insert(forwarding, ZAddress::offset(from_addr), to_addr, cursor);
+}
+
+ZRelocateQueue::ZRelocateQueue() :
+    _lock(),
+    _queue(),
+    _nworkers(0),
+    _nsynchronized(0),
+    _synchronize(false),
+    _needs_attention(0) {}
+
+bool ZRelocateQueue::needs_attention() const {
+  return Atomic::load(&_needs_attention) != 0;
+}
+
+void ZRelocateQueue::inc_needs_attention() {
+  const int needs_attention = Atomic::add(&_needs_attention, 1);
+  assert(needs_attention == 1 || needs_attention == 2, "Invalid state");
+}
+
+void ZRelocateQueue::dec_needs_attention() {
+  const int needs_attention = Atomic::sub(&_needs_attention, 1);
+  assert(needs_attention == 0 || needs_attention == 1, "Invalid state");
+}
+
+void ZRelocateQueue::join(uint nworkers) {
+  assert(nworkers != 0, "Must request at least one worker");
+  assert(_nworkers == 0, "Invalid state");
+  assert(_nsynchronized == 0, "Invalid state");
+
+  log_debug(gc, reloc)("Joining workers: %u", nworkers);
+
+  _nworkers = nworkers;
+}
+
+void ZRelocateQueue::resize_workers(uint nworkers) {
+  assert(nworkers != 0, "Must request at least one worker");
+  assert(_nworkers == 0, "Invalid state");
+  assert(_nsynchronized == 0, "Invalid state");
+
+  log_debug(gc, reloc)("Resize workers: %u", nworkers);
+
+  ZLocker<ZConditionLock> locker(&_lock);
+  _nworkers = nworkers;
+}
+
+void ZRelocateQueue::leave() {
+  ZLocker<ZConditionLock> locker(&_lock);
+  _nworkers--;
+
+  assert(_nsynchronized <= _nworkers, "_nsynchronized: %u _nworkers: %u", _nsynchronized, _nworkers);
+
+  log_debug(gc, reloc)("Leaving workers: left: %u _synchronize: %d _nsynchronized: %u", _nworkers, _synchronize, _nsynchronized);
+
+  // Prune done forwardings
+  const bool forwardings_done = prune();
+
+  // Check if all workers synchronized
+  const bool last_synchronized = _synchronize && _nworkers == _nsynchronized;
+
+  if (forwardings_done || last_synchronized) {
+    _lock.notify_all();
+  }
+}
+
+void ZRelocateQueue::add_and_wait(ZForwarding* forwarding) {
+  ZStatTimer timer(ZCriticalPhaseRelocationStall);
+  ZLocker<ZConditionLock> locker(&_lock);
+
+  if (forwarding->is_done()) {
+    return;
+  }
+
+  _queue.append(forwarding);
+  if (_queue.length() == 1) {
+    // Queue became non-empty
+    inc_needs_attention();
+    _lock.notify_all();
+  }
+
+  while (!forwarding->is_done()) {
+    _lock.wait();
+  }
+}
+
+bool ZRelocateQueue::prune() {
+  if (_queue.is_empty()) {
+    return false;
+  }
+
+  bool done = false;
+
+  for (int i = 0; i < _queue.length();) {
+    const ZForwarding* const forwarding = _queue.at(i);
+    if (forwarding->is_done()) {
+      done = true;
+
+      _queue.delete_at(i);
+    } else {
+      i++;
+    }
+  }
+
+  if (_queue.is_empty()) {
+    dec_needs_attention();
+  }
+
+  return done;
+}
+
+ZForwarding* ZRelocateQueue::prune_and_claim() {
+  if (prune()) {
+    _lock.notify_all();
+  }
+
+  for (int i = 0; i < _queue.length(); i++) {
+    ZForwarding* const forwarding = _queue.at(i);
+    if (forwarding->claim()) {
+      return forwarding;
+    }
+  }
+
+  return nullptr;
+}
+
+class ZRelocateQueueSynchronizeThread {
+private:
+  ZRelocateQueue* const _queue;
+
+public:
+  ZRelocateQueueSynchronizeThread(ZRelocateQueue* queue) :
+      _queue(queue) {
+    _queue->synchronize_thread();
+  }
+
+  ~ZRelocateQueueSynchronizeThread() {
+    _queue->desynchronize_thread();
+  }
+};
+
+void ZRelocateQueue::synchronize_thread() {
+  _nsynchronized++;
+
+  log_debug(gc, reloc)("Synchronize worker _nsynchronized %u", _nsynchronized);
+
+  assert(_nsynchronized <= _nworkers, "_nsynchronized: %u _nworkers: %u", _nsynchronized, _nworkers);
+  if (_nsynchronized == _nworkers) {
+    // All workers synchronized
+    _lock.notify_all();
+  }
+}
+
+void ZRelocateQueue::desynchronize_thread() {
+  _nsynchronized--;
+
+  log_debug(gc, reloc)("Desynchronize worker _nsynchronized %u", _nsynchronized);
+
+  assert(_nsynchronized < _nworkers, "_nsynchronized: %u _nworkers: %u", _nsynchronized, _nworkers);
+}
+
+ZForwarding* ZRelocateQueue::synchronize_poll() {
+  // Fast path avoids locking
+  if (!needs_attention()) {
+    return nullptr;
+  }
+
+  // Slow path to get the next forwarding and/or synchronize
+  ZLocker<ZConditionLock> locker(&_lock);
+
+  {
+    ZForwarding* const forwarding = prune_and_claim();
+    if (forwarding != nullptr) {
+      // Don't become synchronized while there are elements in the queue
+      return forwarding;
+    }
+  }
+
+  if (!_synchronize) {
+    return nullptr;
+  }
+
+  ZRelocateQueueSynchronizeThread rqst(this);
+
+  do {
+    _lock.wait();
+
+    ZForwarding* const forwarding = prune_and_claim();
+    if (forwarding != nullptr) {
+      return forwarding;
+    }
+  } while (_synchronize);
+
+  return nullptr;
+}
+
+void ZRelocateQueue::clear() {
+  assert(_nworkers == 0, "Invalid state");
+
+  if (_queue.is_empty()) {
+    return;
+  }
+
+  ZArrayIterator<ZForwarding*> iter(&_queue);
+  for (ZForwarding* forwarding; iter.next(&forwarding);) {
+    assert(forwarding->is_done(), "All should be done");
+  }
+
+  assert(false, "Clear was not empty");
+
+  _queue.clear();
+  dec_needs_attention();
+}
+
+void ZRelocateQueue::synchronize() {
+  ZLocker<ZConditionLock> locker(&_lock);
+  _synchronize = true;
+
+  inc_needs_attention();
+
+  log_debug(gc, reloc)("Synchronize all workers 1 _nworkers: %u _nsynchronized: %u", _nworkers, _nsynchronized);
+
+  while (_nworkers != _nsynchronized) {
+    _lock.wait();
+    log_debug(gc, reloc)("Synchronize all workers 2 _nworkers: %u _nsynchronized: %u", _nworkers, _nsynchronized);
+  }
+}
+
+void ZRelocateQueue::desynchronize() {
+  ZLocker<ZConditionLock> locker(&_lock);
+  _synchronize = false;
+
+  log_debug(gc, reloc)("Desynchronize all workers _nworkers: %u _nsynchronized: %u", _nworkers, _nsynchronized);
+
+  assert(_nsynchronized <= _nworkers, "_nsynchronized: %u _nworkers: %u", _nsynchronized, _nworkers);
+
+  dec_needs_attention();
+
+  _lock.notify_all();
+}
+
+ZRelocate::ZRelocate(ZGeneration* generation) :
+    _generation(generation),
+    _queue() {}
+
+ZWorkers* ZRelocate::workers() const {
+  return _generation->workers();
+}
+
+void ZRelocate::start() {
+  _queue.join(workers()->active_workers());
+}
+
+void ZRelocate::add_remset(volatile zpointer* p) {
+  ZGeneration::young()->remember(p);
+}
+
+static zaddress relocate_object_inner(ZForwarding* forwarding, zaddress from_addr, ZForwardingCursor* cursor) {
   assert(ZHeap::heap()->is_object_live(from_addr), "Should be live");
 
   // Allocate object
   const size_t size = ZUtils::object_size(from_addr);
-  const uintptr_t to_addr = ZHeap::heap()->alloc_object_for_relocation(size);
-  if (to_addr == 0) {
+
+  ZAllocatorForRelocation* allocator = ZAllocator::relocation(forwarding->to_age());
+
+  const zaddress to_addr = allocator->alloc_object(size);
+
+  if (is_null(to_addr)) {
     // Allocation failed
-    return 0;
+    return zaddress::null;
   }
 
   // Copy object
   ZUtils::object_copy_disjoint(from_addr, to_addr, size);
 
   // Insert forwarding
-  const uintptr_t to_addr_final = forwarding_insert(forwarding, from_addr, to_addr, cursor);
+  const zaddress to_addr_final = forwarding_insert(forwarding, from_addr, to_addr, cursor);
+
   if (to_addr_final != to_addr) {
     // Already relocated, try undo allocation
-    ZHeap::heap()->undo_alloc_object_for_relocation(to_addr, size);
+    allocator->undo_alloc_object(to_addr, size);
   }
 
   return to_addr_final;
 }
 
-uintptr_t ZRelocate::relocate_object(ZForwarding* forwarding, uintptr_t from_addr) const {
+zaddress ZRelocate::relocate_object(ZForwarding* forwarding, zaddress_unsafe from_addr) {
   ZForwardingCursor cursor;
 
   // Lookup forwarding
-  uintptr_t to_addr = forwarding_find(forwarding, from_addr, &cursor);
-  if (to_addr != 0) {
+  zaddress to_addr = forwarding_find(forwarding, from_addr, &cursor);
+  if (!is_null(to_addr)) {
     // Already relocated
     return to_addr;
   }
 
   // Relocate object
-  if (forwarding->retain_page()) {
-    to_addr = relocate_object_inner(forwarding, from_addr, &cursor);
+  if (forwarding->retain_page(&_queue)) {
+    assert(_generation->is_phase_relocate(), "Must be");
+    to_addr = relocate_object_inner(forwarding, safe(from_addr), &cursor);
     forwarding->release_page();
 
-    if (to_addr != 0) {
+    if (!is_null(to_addr)) {
       // Success
       return to_addr;
     }
 
-    // Failed to relocate object. Wait for a worker thread to complete
-    // relocation of this page, and then forward the object. If the GC
-    // aborts the relocation phase before the page has been relocated,
-    // then wait return false and we just forward the object in-place.
-    if (!forwarding->wait_page_released()) {
-      // Forward object in-place
-      return forwarding_insert(forwarding, from_addr, from_addr, &cursor);
-    }
+    // Failed to relocate object. Signal and wait for a worker thread to
+    // complete relocation of this page, and then forward the object.
+    _queue.add_and_wait(forwarding);
   }
 
   // Forward object
   return forward_object(forwarding, from_addr);
 }
 
-uintptr_t ZRelocate::forward_object(ZForwarding* forwarding, uintptr_t from_addr) const {
+zaddress ZRelocate::forward_object(ZForwarding* forwarding, zaddress_unsafe from_addr) {
   ZForwardingCursor cursor;
-  const uintptr_t to_addr = forwarding_find(forwarding, from_addr, &cursor);
-  assert(to_addr != 0, "Should be forwarded");
+  const zaddress to_addr = forwarding_find(forwarding, from_addr, &cursor);
+  assert(!is_null(to_addr), "Should be forwarded: " PTR_FORMAT, untype(from_addr));
   return to_addr;
 }
 
-static ZPage* alloc_page(const ZForwarding* forwarding) {
+static ZPage* alloc_page(ZAllocatorForRelocation* allocator, ZPageType type, size_t size) {
   if (ZStressRelocateInPlace) {
     // Simulate failure to allocate a new page. This will
     // cause the page being relocated to be relocated in-place.
-    return NULL;
+    return nullptr;
   }
 
   ZAllocationFlags flags;
   flags.set_non_blocking();
-  flags.set_worker_relocation();
-  return ZHeap::heap()->alloc_page(forwarding->type(), forwarding->size(), flags);
+  flags.set_gc_relocation();
+
+  return allocator->alloc_page_for_relocation(type, size, flags);
 }
 
-static void free_page(ZPage* page) {
-  ZHeap::heap()->free_page(page, true /* reclaimed */);
-}
+static void retire_target_page(ZGeneration* generation, ZPage* page) {
+  if (generation->is_young() && page->is_old()) {
+    generation->increase_promoted(page->used());
+  } else {
+    generation->increase_compacted(page->used());
+  }
 
-static bool should_free_target_page(ZPage* page) {
   // Free target page if it is empty. We can end up with an empty target
   // page if we allocated a new target page, and then lost the race to
   // relocate the remaining objects, leaving the target page empty when
   // relocation completed.
-  return page != NULL && page->top() == page->start();
+  if (page->used() == 0) {
+    ZHeap::heap()->free_page(page);
+  }
 }
 
 class ZRelocateSmallAllocator {
 private:
-  volatile size_t _in_place_count;
+  ZGeneration* const _generation;
+  volatile size_t    _in_place_count;
 
 public:
-  ZRelocateSmallAllocator() :
+  ZRelocateSmallAllocator(ZGeneration* generation) :
+      _generation(generation),
       _in_place_count(0) {}
 
-  ZPage* alloc_target_page(ZForwarding* forwarding, ZPage* target) {
-    ZPage* const page = alloc_page(forwarding);
-    if (page == NULL) {
+  ZPage* alloc_and_retire_target_page(ZForwarding* forwarding, ZPage* target) {
+    ZAllocatorForRelocation* const allocator = ZAllocator::relocation(forwarding->to_age());
+    ZPage* const page = alloc_page(allocator, forwarding->type(), forwarding->size());
+    if (page == nullptr) {
       Atomic::inc(&_in_place_count);
     }
 
+    if (target != nullptr) {
+      // Retire the old target page
+      retire_target_page(_generation, target);
+    }
+
     return page;
   }
 
@@ -172,20 +460,16 @@ public:
   }
 
   void free_target_page(ZPage* page) {
-    if (should_free_target_page(page)) {
-      free_page(page);
+    if (page != nullptr) {
+      retire_target_page(_generation, page);
     }
   }
 
-  void free_relocated_page(ZPage* page) {
-    free_page(page);
+  zaddress alloc_object(ZPage* page, size_t size) const {
+    return (page != nullptr) ? page->alloc_object(size) : zaddress::null;
   }
 
-  uintptr_t alloc_object(ZPage* page, size_t size) const {
-    return (page != NULL) ? page->alloc_object(size) : 0;
-  }
-
-  void undo_alloc_object(ZPage* page, uintptr_t addr, size_t size) const {
+  void undo_alloc_object(ZPage* page, zaddress addr, size_t size) const {
     page->undo_alloc_object(addr, size);
   }
 
@@ -196,25 +480,37 @@ public:
 
 class ZRelocateMediumAllocator {
 private:
-  ZConditionLock      _lock;
-  ZPage*              _shared;
-  bool                _in_place;
-  volatile size_t     _in_place_count;
+  ZGeneration* const _generation;
+  ZConditionLock     _lock;
+  ZPage*             _shared[ZAllocator::_relocation_allocators];
+  bool               _in_place;
+  volatile size_t    _in_place_count;
 
 public:
-  ZRelocateMediumAllocator() :
+  ZRelocateMediumAllocator(ZGeneration* generation) :
+      _generation(generation),
       _lock(),
-      _shared(NULL),
+      _shared(),
       _in_place(false),
       _in_place_count(0) {}
 
   ~ZRelocateMediumAllocator() {
-    if (should_free_target_page(_shared)) {
-      free_page(_shared);
+    for (uint i = 0; i < ZAllocator::_relocation_allocators; ++i) {
+      if (_shared[i] != nullptr) {
+        retire_target_page(_generation, _shared[i]);
+      }
     }
   }
 
-  ZPage* alloc_target_page(ZForwarding* forwarding, ZPage* target) {
+  ZPage* shared(ZPageAge age) {
+    return _shared[static_cast<uint>(age) - 1];
+  }
+
+  void set_shared(ZPageAge age, ZPage* page) {
+    _shared[static_cast<uint>(age) - 1] = page;
+  }
+
+  ZPage* alloc_and_retire_target_page(ZForwarding* forwarding, ZPage* target) {
     ZLocker<ZConditionLock> locker(&_lock);
 
     // Wait for any ongoing in-place relocation to complete
@@ -226,25 +522,34 @@ public:
     // current target page. The shared page will be different from the
     // current target page if another thread shared a page, or allocated
     // a new page.
-    if (_shared == target) {
-      _shared = alloc_page(forwarding);
-      if (_shared == NULL) {
+    const ZPageAge to_age = forwarding->to_age();
+    if (shared(to_age) == target) {
+      ZAllocatorForRelocation* const allocator = ZAllocator::relocation(forwarding->to_age());
+      ZPage* const to_page = alloc_page(allocator, forwarding->type(), forwarding->size());
+      set_shared(to_age, to_page);
+      if (to_page == nullptr) {
         Atomic::inc(&_in_place_count);
         _in_place = true;
       }
+
+      // This thread is responsible for retiring the shared target page
+      if (target != nullptr) {
+        retire_target_page(_generation, target);
+      }
     }
 
-    return _shared;
+    return shared(to_age);
   }
 
   void share_target_page(ZPage* page) {
+    const ZPageAge age = page->age();
+
     ZLocker<ZConditionLock> locker(&_lock);
-
     assert(_in_place, "Invalid state");
-    assert(_shared == NULL, "Invalid state");
-    assert(page != NULL, "Invalid page");
+    assert(shared(age) == nullptr, "Invalid state");
+    assert(page != nullptr, "Invalid page");
 
-    _shared = page;
+    set_shared(age, page);
     _in_place = false;
 
     _lock.notify_all();
@@ -254,15 +559,11 @@ public:
     // Does nothing
   }
 
-  void free_relocated_page(ZPage* page) {
-    free_page(page);
+  zaddress alloc_object(ZPage* page, size_t size) const {
+    return (page != nullptr) ? page->alloc_object_atomic(size) : zaddress::null;
   }
 
-  uintptr_t alloc_object(ZPage* page, size_t size) const {
-    return (page != NULL) ? page->alloc_object_atomic(size) : 0;
-  }
-
-  void undo_alloc_object(ZPage* page, uintptr_t addr, size_t size) const {
+  void undo_alloc_object(ZPage* page, zaddress addr, size_t size) const {
     page->undo_alloc_object_atomic(addr, size);
   }
 
@@ -272,148 +573,746 @@ public:
 };
 
 template <typename Allocator>
-class ZRelocateClosure : public ObjectClosure {
+class ZRelocateWork : public StackObj {
 private:
-  Allocator* const _allocator;
-  ZForwarding*     _forwarding;
-  ZPage*           _target;
+  Allocator* const   _allocator;
+  ZForwarding*       _forwarding;
+  ZPage*             _target[ZAllocator::_relocation_allocators];
+  ZGeneration* const _generation;
+  size_t             _other_promoted;
+  size_t             _other_compacted;
 
-  bool relocate_object(uintptr_t from_addr) const {
+  ZPage* target(ZPageAge age) {
+    return _target[static_cast<uint>(age) - 1];
+  }
+
+  void set_target(ZPageAge age, ZPage* page) {
+    _target[static_cast<uint>(age) - 1] = page;
+  }
+
+  size_t object_alignment() const {
+    return (size_t)1 << _forwarding->object_alignment_shift();
+  }
+
+  void increase_other_forwarded(size_t unaligned_object_size) {
+    const size_t aligned_size = align_up(unaligned_object_size, object_alignment());
+    if (_forwarding->is_promotion()) {
+      _other_promoted += aligned_size;
+    } else {
+      _other_compacted += aligned_size;
+    }
+  }
+
+  zaddress try_relocate_object_inner(zaddress from_addr) {
     ZForwardingCursor cursor;
 
+    const size_t size = ZUtils::object_size(from_addr);
+    ZPage* const to_page = target(_forwarding->to_age());
+
     // Lookup forwarding
-    if (forwarding_find(_forwarding, from_addr, &cursor) != 0) {
-      // Already relocated
-      return true;
+    {
+      const zaddress to_addr = forwarding_find(_forwarding, from_addr, &cursor);
+      if (!is_null(to_addr)) {
+        // Already relocated
+        increase_other_forwarded(size);
+        return to_addr;
+      }
     }
 
     // Allocate object
-    const size_t size = ZUtils::object_size(from_addr);
-    const uintptr_t to_addr = _allocator->alloc_object(_target, size);
-    if (to_addr == 0) {
+    const zaddress allocated_addr = _allocator->alloc_object(to_page, size);
+    if (is_null(allocated_addr)) {
       // Allocation failed
-      return false;
+      return zaddress::null;
     }
 
     // Copy object. Use conjoint copying if we are relocating
-    // in-place and the new object overlapps with the old object.
-    if (_forwarding->in_place() && to_addr + size > from_addr) {
-      ZUtils::object_copy_conjoint(from_addr, to_addr, size);
+    // in-place and the new object overlaps with the old object.
+    if (_forwarding->in_place_relocation() && allocated_addr + size > from_addr) {
+      ZUtils::object_copy_conjoint(from_addr, allocated_addr, size);
     } else {
-      ZUtils::object_copy_disjoint(from_addr, to_addr, size);
+      ZUtils::object_copy_disjoint(from_addr, allocated_addr, size);
     }
 
     // Insert forwarding
-    if (forwarding_insert(_forwarding, from_addr, to_addr, &cursor) != to_addr) {
+    const zaddress to_addr = forwarding_insert(_forwarding, from_addr, allocated_addr, &cursor);
+    if (to_addr != allocated_addr) {
       // Already relocated, undo allocation
-      _allocator->undo_alloc_object(_target, to_addr, size);
+      _allocator->undo_alloc_object(to_page, to_addr, size);
+      increase_other_forwarded(size);
     }
 
+    return to_addr;
+  }
+
+  void update_remset_old_to_old(zaddress from_addr, zaddress to_addr) const {
+    // Old-to-old relocation - move existing remset bits
+
+    // If this is called for an in-place relocated page, then this code has the
+    // responsibility to clear the old remset bits. Extra care is needed because:
+    //
+    // 1) The to-object copy can overlap with the from-object copy
+    // 2) Remset bits of old objects need to be cleared
+    //
+    // A watermark is used to keep track of how far the old remset bits have been removed.
+
+    const bool in_place = _forwarding->in_place_relocation();
+    ZPage* const from_page = _forwarding->page();
+    const uintptr_t from_local_offset = from_page->local_offset(from_addr);
+
+    // Note: even with in-place relocation, the to_page could be another page
+    ZPage* const to_page = ZHeap::heap()->page(to_addr);
+
+    // Uses _relaxed version to handle that in-place relocation resets _top
+    assert(ZHeap::heap()->is_in_page_relaxed(from_page, from_addr), "Must be");
+    assert(to_page->is_in(to_addr), "Must be");
+
+
+    // Read the size from the to-object, since the from-object
+    // could have been overwritten during in-place relocation.
+    const size_t size = ZUtils::object_size(to_addr);
+
+    // If a young generation collection started while the old generation
+    // relocated  objects, the remember set bits were flipped from "current"
+    // to "previous".
+    //
+    // We need to select the correct remembered sets bitmap to ensure that the
+    // old remset bits are found.
+    //
+    // Note that if the young generation marking (remset scanning) finishes
+    // before the old generation relocation has relocated this page, then the
+    // young generation will visit this page's previous remembered set bits and
+    // moved them over to the current bitmap.
+    //
+    // If the young generation runs multiple cycles while the old generation is
+    // relocating, then the first cycle will have consume the the old remset,
+    // bits and moved associated objects to a new old page. The old relocation
+    // could find either the the two bitmaps. So, either it will find the original
+    // remset bits for the page, or it will find an empty bitmap for the page. It
+    // doesn't matter for correctness, because the young generation marking has
+    // already taken care of the bits.
+
+    const bool active_remset_is_current = ZGeneration::old()->active_remset_is_current();
+
+    // When in-place relocation is done and the old remset bits are located in
+    // the bitmap that is going to be used for the new remset bits, then we
+    // need to clear the old bits before the new bits are inserted.
+    const bool iterate_current_remset = active_remset_is_current && !in_place;
+
+    BitMap::Iterator iter = iterate_current_remset
+        ? from_page->remset_iterator_limited_current(from_local_offset, size)
+        : from_page->remset_iterator_limited_previous(from_local_offset, size);
+
+    for (BitMap::idx_t field_bit : iter) {
+      const uintptr_t field_local_offset = ZRememberedSet::to_offset(field_bit);
+
+      // Add remset entry in the to-page
+      const uintptr_t offset = field_local_offset - from_local_offset;
+      const zaddress to_field = to_addr + offset;
+      log_trace(gc, reloc)("Remember: from: " PTR_FORMAT " to: " PTR_FORMAT " current: %d marking: %d page: " PTR_FORMAT " remset: " PTR_FORMAT,
+          untype(from_page->start() + field_local_offset), untype(to_field), active_remset_is_current, ZGeneration::young()->is_phase_mark(), p2i(to_page), p2i(to_page->remset_current()));
+
+      volatile zpointer* const p = (volatile zpointer*)to_field;
+
+      if (ZGeneration::young()->is_phase_mark()) {
+        // Young generation remembered set scanning needs to know about this
+        // field. It will take responsibility to add a new remember set entry if needed.
+        _forwarding->relocated_remembered_fields_register(p);
+      } else {
+        to_page->remember(p);
+        if (in_place) {
+          assert(to_page->is_remembered(p), "p: " PTR_FORMAT, p2i(p));
+        }
+      }
+    }
+  }
+
+  static bool add_remset_if_young(volatile zpointer* p, zaddress addr) {
+    if (ZHeap::heap()->is_young(addr)) {
+      ZRelocate::add_remset(p);
+      return true;
+    }
+
+    return false;
+  }
+
+  static void update_remset_promoted_filter_and_remap_per_field(volatile zpointer* p) {
+    const zpointer ptr = Atomic::load(p);
+
+    assert(ZPointer::is_old_load_good(ptr), "Should be at least old load good: " PTR_FORMAT, untype(ptr));
+
+    if (ZPointer::is_store_good(ptr)) {
+      // Already has a remset entry
+      return;
+    }
+
+    if (ZPointer::is_load_good(ptr)) {
+      if (!is_null_any(ptr)) {
+        const zaddress addr = ZPointer::uncolor(ptr);
+        add_remset_if_young(p, addr);
+      }
+      // No need to remap it is already load good
+      return;
+    }
+
+    if (is_null_any(ptr)) {
+      // Eagerly remap to skip adding a remset entry just to get deferred remapping
+      ZBarrier::remap_young_relocated(p, ptr);
+      return;
+    }
+
+    const zaddress_unsafe addr_unsafe = ZPointer::uncolor_unsafe(ptr);
+    ZForwarding* const forwarding = ZGeneration::young()->forwarding(addr_unsafe);
+
+    if (forwarding == nullptr) {
+      // Object isn't being relocated
+      const zaddress addr = safe(addr_unsafe);
+      if (!add_remset_if_young(p, addr)) {
+        // Not young - eagerly remap to skip adding a remset entry just to get deferred remapping
+        ZBarrier::remap_young_relocated(p, ptr);
+      }
+      return;
+    }
+
+    const zaddress addr = forwarding->find(addr_unsafe);
+
+    if (!is_null(addr)) {
+      // Object has already been relocated
+      if (!add_remset_if_young(p, addr)) {
+        // Not young - eagerly remap to skip adding a remset entry just to get deferred remapping
+        ZBarrier::remap_young_relocated(p, ptr);
+      }
+      return;
+    }
+
+    // Object has not been relocated yet
+    // Don't want to eagerly relocate objects, so just add a remset
+    ZRelocate::add_remset(p);
+    return;
+  }
+
+  void update_remset_promoted(zaddress to_addr) const {
+    ZIterator::basic_oop_iterate(to_oop(to_addr), update_remset_promoted_filter_and_remap_per_field);
+  }
+
+  void update_remset_for_fields(zaddress from_addr, zaddress to_addr) const {
+    if (_forwarding->to_age() != ZPageAge::old) {
+      // No remembered set in young pages
+      return;
+    }
+
+    // Need to deal with remset when moving objects to the old generation
+    if (_forwarding->from_age() == ZPageAge::old) {
+      update_remset_old_to_old(from_addr, to_addr);
+      return;
+    }
+
+    // Normal promotion
+    update_remset_promoted(to_addr);
+  }
+
+  bool try_relocate_object(zaddress from_addr) {
+    const zaddress to_addr = try_relocate_object_inner(from_addr);
+
+    if (is_null(to_addr)) {
+      return false;
+    }
+
+    update_remset_for_fields(from_addr, to_addr);
+
     return true;
   }
 
-  virtual void do_object(oop obj) {
-    const uintptr_t addr = ZOop::to_address(obj);
+  void start_in_place_relocation_prepare_remset(ZPage* from_page) {
+    if (_forwarding->from_age() != ZPageAge::old) {
+      // Only old pages have use remset bits
+      return;
+    }
+
+    if (ZGeneration::old()->active_remset_is_current()) {
+      // We want to iterate over and clear the remset bits of the from-space page,
+      // and insert current bits in the to-space page. However, with in-place
+      // relocation, the from-space and to-space pages are the same. Clearing
+      // is destructive, and is difficult to perform before or during the iteration.
+      // However, clearing of the current bits has to be done before exposing the
+      // to-space objects in the forwarding table.
+      //
+      // To solve this tricky dependency problem, we start by stashing away the
+      // current bits in the previous bits, and clearing the current bits
+      // (implemented by swapping the bits). This way, the current bits are
+      // cleared before copying the objects (like a normal to-space page),
+      // and the previous bits are representing a copy of the current bits
+      // of the from-space page, and are used for iteration.
+      from_page->swap_remset_bitmaps();
+    }
+  }
+
+  ZPage* start_in_place_relocation(zoffset relocated_watermark) {
+    _forwarding->in_place_relocation_claim_page();
+    _forwarding->in_place_relocation_start(relocated_watermark);
+
+    ZPage* const from_page = _forwarding->page();
+
+    const ZPageAge to_age = _forwarding->to_age();
+    const bool promotion = _forwarding->is_promotion();
+
+    // Promotions happen through a new cloned page
+    ZPage* const to_page = promotion ? from_page->clone_limited() : from_page;
+    to_page->reset(to_age, ZPageResetType::InPlaceRelocation);
+
+    // Clear remset bits for all objects that were relocated
+    // before this page became an in-place relocated page.
+    start_in_place_relocation_prepare_remset(from_page);
+
+    if (promotion) {
+      // Register the the promotion
+      ZGeneration::young()->in_place_relocate_promote(from_page, to_page);
+      ZGeneration::young()->register_in_place_relocate_promoted(from_page);
+    }
+
+    return to_page;
+  }
+
+  void relocate_object(oop obj) {
+    const zaddress addr = to_zaddress(obj);
     assert(ZHeap::heap()->is_object_live(addr), "Should be live");
 
-    while (!relocate_object(addr)) {
+    while (!try_relocate_object(addr)) {
       // Allocate a new target page, or if that fails, use the page being
       // relocated as the new target, which will cause it to be relocated
       // in-place.
-      _target = _allocator->alloc_target_page(_forwarding, _target);
-      if (_target != NULL) {
+      const ZPageAge to_age = _forwarding->to_age();
+      ZPage* to_page = _allocator->alloc_and_retire_target_page(_forwarding, target(to_age));
+      set_target(to_age, to_page);
+      if (to_page != nullptr) {
         continue;
       }
 
-      // Claim the page being relocated to block other threads from accessing
-      // it, or its forwarding table, until it has been released (relocation
-      // completed).
-      _target = _forwarding->claim_page();
-      _target->reset_for_in_place_relocation();
-      _forwarding->set_in_place();
+      // Start in-place relocation to block other threads from accessing
+      // the page, or its forwarding table, until it has been released
+      // (relocation completed).
+      to_page = start_in_place_relocation(ZAddress::offset(addr));
+      set_target(to_age, to_page);
     }
   }
 
 public:
-  ZRelocateClosure(Allocator* allocator) :
+  ZRelocateWork(Allocator* allocator, ZGeneration* generation) :
       _allocator(allocator),
-      _forwarding(NULL),
-      _target(NULL) {}
+      _forwarding(nullptr),
+      _target(),
+      _generation(generation),
+      _other_promoted(0),
+      _other_compacted(0) {}
 
-  ~ZRelocateClosure() {
-    _allocator->free_target_page(_target);
+  ~ZRelocateWork() {
+    for (uint i = 0; i < ZAllocator::_relocation_allocators; ++i) {
+      _allocator->free_target_page(_target[i]);
+    }
+    // Report statistics on-behalf of non-worker threads
+    _generation->increase_promoted(_other_promoted);
+    _generation->increase_compacted(_other_compacted);
+  }
+
+  bool active_remset_is_current() const {
+    // Normal old-to-old relocation can treat the from-page remset as a
+    // read-only copy, and then copy over the appropriate remset bits to the
+    // cleared to-page's 'current' remset bitmap.
+    //
+    // In-place relocation is more complicated. Since, the same page is both
+    // a from-page and a to-page, we need to remove the old remset bits, and
+    // add remset bits that corresponds to the new locations of the relocated
+    // objects.
+    //
+    // Depending on how long ago (in terms of number of young GC's and the
+    // current young GC's phase), the page was allocated, the active
+    // remembered set will be in either the 'current' or 'previous' bitmap.
+    //
+    // If the active bits are in the 'previous' bitmap, we know that the
+    // 'current' bitmap was cleared at some earlier point in time, and we can
+    // simply set new bits in 'current' bitmap, and later when relocation has
+    // read all the old remset bits, we could just clear the 'previous' remset
+    // bitmap.
+    //
+    // If, on the other hand, the active bits are in the 'current' bitmap, then
+    // that bitmap will be used to both read the old remset bits, and the
+    // destination for the remset bits that we copy when an object is copied
+    // to it's new location within the page. We need to *carefully* remove all
+    // all old remset bits, without clearing out the newly set bits.
+    return ZGeneration::old()->active_remset_is_current();
+  }
+
+  void clear_remset_before_reuse(ZPage* page, bool in_place) {
+    if (_forwarding->from_age() != ZPageAge::old) {
+      // No remset bits
+      return;
+    }
+
+    if (in_place) {
+      // Clear 'previous' remset bits. For in-place relocated pages, the previous
+      // remset bits are always used, even when active_remset_is_current().
+      page->clear_remset_previous();
+
+      return;
+    }
+
+    // Normal relocate
+
+    // Clear active remset bits
+    if (active_remset_is_current()) {
+      page->clear_remset_current();
+    } else {
+      page->clear_remset_previous();
+    }
+
+    // Verify that inactive remset bits are all cleared
+    if (active_remset_is_current()) {
+      page->verify_remset_cleared_previous();
+    } else {
+      page->verify_remset_cleared_current();
+    }
+  }
+
+  void finish_in_place_relocation() {
+    // We are done with the from_space copy of the page
+    _forwarding->in_place_relocation_finish();
   }
 
   void do_forwarding(ZForwarding* forwarding) {
     _forwarding = forwarding;
 
-    // Check if we should abort
-    if (ZAbort::should_abort()) {
-      _forwarding->abort_page();
-      return;
-    }
+    _forwarding->page()->log_msg(" (relocate page)");
+
+    ZVerify::before_relocation(_forwarding);
 
     // Relocate objects
-    _forwarding->object_iterate(this);
+    _forwarding->object_iterate([&](oop obj) { relocate_object(obj); });
+
+    ZVerify::after_relocation(_forwarding);
 
     // Verify
     if (ZVerifyForwarding) {
       _forwarding->verify();
     }
 
+    _generation->increase_freed(_forwarding->page()->size());
+
+    // Deal with in-place relocation
+    const bool in_place = _forwarding->in_place_relocation();
+    if (in_place) {
+      finish_in_place_relocation();
+    }
+
+    // Old from-space pages need to deal with remset bits
+    if (_forwarding->from_age() == ZPageAge::old) {
+      _forwarding->relocated_remembered_fields_after_relocate();
+    }
+
     // Release relocated page
     _forwarding->release_page();
 
-    if (_forwarding->in_place()) {
-      // The relocated page has been relocated in-place and should not
-      // be freed. Keep it as target page until it is full, and offer to
-      // share it with other worker threads.
-      _allocator->share_target_page(_target);
-    } else {
-      // Detach and free relocated page
+    if (in_place) {
+      // Wait for all other threads to call release_page
       ZPage* const page = _forwarding->detach_page();
-      _allocator->free_relocated_page(page);
+
+      // Ensure that previous remset bits are cleared
+      clear_remset_before_reuse(page, true /* in_place */);
+
+      page->log_msg(" (relocate page done in-place)");
+
+      // Different pages when promoting
+      ZPage* const target_page = target(_forwarding->to_age());
+      _allocator->share_target_page(target_page);
+
+    } else {
+      // Wait for all other threads to call release_page
+      ZPage* const page = _forwarding->detach_page();
+
+      // Ensure that all remset bits are cleared
+      // Note: cleared after detach_page, when we know that
+      // the young generation isn't scanning the remset.
+      clear_remset_before_reuse(page, false /* in_place */);
+
+      page->log_msg(" (relocate page done normal)");
+
+      // Free page
+      ZHeap::heap()->free_page(page);
     }
   }
 };
 
-class ZRelocateTask : public ZTask {
+class ZRelocateStoreBufferInstallBasePointersThreadClosure : public ThreadClosure {
+public:
+  virtual void do_thread(Thread* thread) {
+    JavaThread* const jt = JavaThread::cast(thread);
+    ZStoreBarrierBuffer* buffer = ZThreadLocalData::store_barrier_buffer(jt);
+    buffer->install_base_pointers();
+  }
+};
+
+// Installs the object base pointers (object starts), for the fields written
+// in the store buffer. The code that searches for the object start uses that
+// liveness information stored in the pages. That information is lost when the
+// pages have been relocated and then destroyed.
+class ZRelocateStoreBufferInstallBasePointersTask : public ZTask {
+private:
+  ZJavaThreadsIterator _threads_iter;
+
+public:
+  ZRelocateStoreBufferInstallBasePointersTask(ZGeneration* generation) :
+    ZTask("ZRelocateStoreBufferInstallBasePointersTask"),
+    _threads_iter(generation->id_optional()) {}
+
+  virtual void work() {
+    ZRelocateStoreBufferInstallBasePointersThreadClosure fix_store_buffer_cl;
+    _threads_iter.apply(&fix_store_buffer_cl);
+  }
+};
+
+class ZRelocateTask : public ZRestartableTask {
 private:
   ZRelocationSetParallelIterator _iter;
+  ZGeneration* const             _generation;
+  ZRelocateQueue* const          _queue;
   ZRelocateSmallAllocator        _small_allocator;
   ZRelocateMediumAllocator       _medium_allocator;
 
-  static bool is_small(ZForwarding* forwarding) {
-    return forwarding->type() == ZPageTypeSmall;
-  }
-
 public:
-  ZRelocateTask(ZRelocationSet* relocation_set) :
-      ZTask("ZRelocateTask"),
+  ZRelocateTask(ZRelocationSet* relocation_set, ZRelocateQueue* queue) :
+      ZRestartableTask("ZRelocateTask"),
       _iter(relocation_set),
-      _small_allocator(),
-      _medium_allocator() {}
+      _generation(relocation_set->generation()),
+      _queue(queue),
+      _small_allocator(_generation),
+      _medium_allocator(_generation) {}
 
   ~ZRelocateTask() {
-    ZStatRelocation::set_at_relocate_end(_small_allocator.in_place_count(),
-                                         _medium_allocator.in_place_count());
+    _generation->stat_relocation()->at_relocate_end(_small_allocator.in_place_count(), _medium_allocator.in_place_count());
   }
 
   virtual void work() {
-    ZRelocateClosure<ZRelocateSmallAllocator> small(&_small_allocator);
-    ZRelocateClosure<ZRelocateMediumAllocator> medium(&_medium_allocator);
+    ZRelocateWork<ZRelocateSmallAllocator> small(&_small_allocator, _generation);
+    ZRelocateWork<ZRelocateMediumAllocator> medium(&_medium_allocator, _generation);
 
-    for (ZForwarding* forwarding; _iter.next(&forwarding);) {
-      if (is_small(forwarding)) {
+    const auto do_forwarding = [&](ZForwarding* forwarding) {
+      ZPage* const page = forwarding->page();
+      if (page->is_small()) {
         small.do_forwarding(forwarding);
       } else {
         medium.do_forwarding(forwarding);
       }
+
+      // Absolute last thing done while relocating a page.
+      //
+      // We don't use the SuspendibleThreadSet when relocating pages.
+      // Instead the ZRelocateQueue is used as a pseudo STS joiner/leaver.
+      //
+      // After the mark_done call a safepointing could be completed and a
+      // new GC phase could be entered.
+      forwarding->mark_done();
+    };
+
+    const auto claim_and_do_forwarding = [&](ZForwarding* forwarding) {
+      if (forwarding->claim()) {
+        do_forwarding(forwarding);
+      }
+    };
+
+    const auto do_forwarding_one_from_iter = [&]() {
+      ZForwarding* forwarding;
+
+      if (_iter.next(&forwarding)) {
+        claim_and_do_forwarding(forwarding);
+        return true;
+      }
+
+      return false;
+    };
+
+    for (;;) {
+      // As long as there are requests in the relocate queue, there are threads
+      // waiting in a VM state that does not allow them to be blocked. The
+      // worker thread needs to finish relocate these pages, and allow the
+      // other threads to continue and proceed to a blocking state. After that,
+      // the worker threads are allowed to safepoint synchronize.
+      for (ZForwarding* forwarding; (forwarding = _queue->synchronize_poll()) != nullptr;) {
+        do_forwarding(forwarding);
+      }
+
+      if (!do_forwarding_one_from_iter()) {
+        // No more work
+        break;
+      }
+
+      if (_generation->should_worker_resize()) {
+        break;
+      }
+    }
+
+    _queue->leave();
+  }
+
+  virtual void resize_workers(uint nworkers) {
+    _queue->resize_workers(nworkers);
+  }
+};
+
+static void remap_and_maybe_add_remset(volatile zpointer* p) {
+  const zpointer ptr = Atomic::load(p);
+
+  if (ZPointer::is_store_good(ptr)) {
+    // Already has a remset entry
+    return;
+  }
+
+  // Remset entries are used for two reasons:
+  // 1) Young marking old-to-young pointer roots
+  // 2) Deferred remapping of stale old-to-young pointers
+  //
+  // This load barrier will up-front perform the remapping of (2),
+  // and the code below only has to make sure we register up-to-date
+  // old-to-young pointers for (1).
+  const zaddress addr = ZBarrier::load_barrier_on_oop_field_preloaded(p, ptr);
+
+  if (is_null(addr)) {
+    // No need for remset entries for null pointers
+    return;
+  }
+
+  if (ZHeap::heap()->is_old(addr)) {
+    // No need for remset entries for pointers to old gen
+    return;
+  }
+
+  ZRelocate::add_remset(p);
+}
+
+class ZRelocateAddRemsetForFlipPromoted : public ZRestartableTask {
+private:
+  ZStatTimerYoung                _timer;
+  ZArrayParallelIterator<ZPage*> _iter;
+
+public:
+  ZRelocateAddRemsetForFlipPromoted(ZArray<ZPage*>* pages) :
+      ZRestartableTask("ZRelocateAddRemsetForFlipPromoted"),
+      _timer(ZSubPhaseConcurrentRelocateRememberedSetFlipPromotedYoung),
+      _iter(pages) {}
+
+  virtual void work() {
+    SuspendibleThreadSetJoiner sts_joiner;
+
+    for (ZPage* page; _iter.next(&page);) {
+      page->object_iterate([&](oop obj) {
+        ZIterator::basic_oop_iterate_safe(obj, remap_and_maybe_add_remset);
+      });
+
+      SuspendibleThreadSet::yield();
+      if (ZGeneration::young()->should_worker_resize()) {
+        return;
+      }
     }
   }
 };
 
 void ZRelocate::relocate(ZRelocationSet* relocation_set) {
-  ZRelocateTask task(relocation_set);
-  _workers->run(&task);
+  {
+    // Install the store buffer's base pointers before the
+    // relocate task destroys the liveness information in
+    // the relocated pages.
+    ZRelocateStoreBufferInstallBasePointersTask buffer_task(_generation);
+    workers()->run(&buffer_task);
+  }
+
+  {
+    ZRelocateTask relocate_task(relocation_set, &_queue);
+    workers()->run(&relocate_task);
+  }
+
+  if (relocation_set->generation()->is_young()) {
+    ZRelocateAddRemsetForFlipPromoted task(relocation_set->flip_promoted_pages());
+    workers()->run(&task);
+  }
+
+  _queue.clear();
+}
+
+ZPageAge ZRelocate::compute_to_age(ZPageAge from_age) {
+  if (from_age == ZPageAge::old) {
+    return ZPageAge::old;
+  }
+
+  const uint age = static_cast<uint>(from_age);
+  if (age >= ZGeneration::young()->tenuring_threshold()) {
+    return ZPageAge::old;
+  }
+
+  return static_cast<ZPageAge>(age + 1);
+}
+
+class ZFlipAgePagesTask : public ZTask {
+private:
+  ZArrayParallelIterator<ZPage*> _iter;
+
+public:
+  ZFlipAgePagesTask(const ZArray<ZPage*>* pages) :
+      ZTask("ZPromotePagesTask"),
+      _iter(pages) {}
+
+  virtual void work() {
+    SuspendibleThreadSetJoiner sts_joiner;
+    ZArray<ZPage*> promoted_pages;
+
+    for (ZPage* prev_page; _iter.next(&prev_page);) {
+      const ZPageAge from_age = prev_page->age();
+      const ZPageAge to_age = ZRelocate::compute_to_age(from_age);
+      assert(from_age != ZPageAge::old, "invalid age for a young collection");
+
+      // Figure out if this is proper promotion
+      const bool promotion = to_age == ZPageAge::old;
+
+      if (promotion) {
+        // Before promoting an object (and before relocate start), we must ensure that all
+        // contained zpointers are store good. The marking code ensures that for non-null
+        // pointers, but null pointers are ignored. This code ensures that even null pointers
+        // are made store good, for the promoted objects.
+        prev_page->object_iterate([&](oop obj) {
+          ZIterator::basic_oop_iterate_safe(obj, ZBarrier::promote_barrier_on_young_oop_field);
+        });
+      }
+
+      // Logging
+      prev_page->log_msg(promotion ? " (flip promoted)" : " (flip survived)");
+
+      // Setup to-space page
+      ZPage* const new_page = promotion ? prev_page->clone_limited_promote_flipped() : prev_page;
+      new_page->reset(to_age, ZPageResetType::FlipAging);
+
+      if (promotion) {
+        ZGeneration::young()->flip_promote(prev_page, new_page);
+        // Defer promoted page registration times the lock is taken
+        promoted_pages.push(prev_page);
+      }
+
+      SuspendibleThreadSet::yield();
+    }
+
+    ZGeneration::young()->register_flip_promoted(promoted_pages);
+  }
+};
+
+void ZRelocate::flip_age_pages(const ZArray<ZPage*>* pages) {
+  ZFlipAgePagesTask flip_age_task(pages);
+  workers()->run(&flip_age_task);
+}
+
+void ZRelocate::synchronize() {
+  _queue.synchronize();
+}
+
+void ZRelocate::desynchronize() {
+  _queue.desynchronize();
+}
+
+ZRelocateQueue* ZRelocate::queue() {
+  return &_queue;
 }
diff --git a/src/hotspot/share/gc/z/zRelocate.hpp b/src/hotspot/share/gc/z/zRelocate.hpp
index 647a640970a..ed54103d53c 100644
--- a/src/hotspot/share/gc/z/zRelocate.hpp
+++ b/src/hotspot/share/gc/z/zRelocate.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,26 +24,81 @@
 #ifndef SHARE_GC_Z_ZRELOCATE_HPP
 #define SHARE_GC_Z_ZRELOCATE_HPP
 
+#include "gc/z/zAddress.hpp"
+#include "gc/z/zPageAge.hpp"
 #include "gc/z/zRelocationSet.hpp"
 
 class ZForwarding;
+class ZGeneration;
 class ZWorkers;
 
+typedef size_t ZForwardingCursor;
+
+class ZRelocateQueue {
+private:
+  ZConditionLock       _lock;
+  ZArray<ZForwarding*> _queue;
+  uint                 _nworkers;
+  uint                 _nsynchronized;
+  bool                 _synchronize;
+  volatile int         _needs_attention;
+
+  bool needs_attention() const;
+  void inc_needs_attention();
+  void dec_needs_attention();
+
+  bool prune();
+  ZForwarding* prune_and_claim();
+
+public:
+  ZRelocateQueue();
+
+  void join(uint nworkers);
+  void resize_workers(uint nworkers);
+  void leave();
+
+  void add_and_wait(ZForwarding* forwarding);
+
+  ZForwarding* synchronize_poll();
+  void synchronize_thread();
+  void desynchronize_thread();
+
+  void clear();
+
+  void synchronize();
+  void desynchronize();
+};
+
 class ZRelocate {
   friend class ZRelocateTask;
 
 private:
-  ZWorkers* const _workers;
+  ZGeneration* const _generation;
+  ZRelocateQueue     _queue;
 
+  ZWorkers* workers() const;
   void work(ZRelocationSetParallelIterator* iter);
 
 public:
-  ZRelocate(ZWorkers* workers);
+  ZRelocate(ZGeneration* generation);
 
-  uintptr_t relocate_object(ZForwarding* forwarding, uintptr_t from_addr) const;
-  uintptr_t forward_object(ZForwarding* forwarding, uintptr_t from_addr) const;
+  void start();
+
+  static void add_remset(volatile zpointer* p);
+
+  static ZPageAge compute_to_age(ZPageAge from_age);
+
+  zaddress relocate_object(ZForwarding* forwarding, zaddress_unsafe from_addr);
+  zaddress forward_object(ZForwarding* forwarding, zaddress_unsafe from_addr);
 
   void relocate(ZRelocationSet* relocation_set);
+
+  void flip_age_pages(const ZArray<ZPage*>* pages);
+
+  void synchronize();
+  void desynchronize();
+
+  ZRelocateQueue* queue();
 };
 
 #endif // SHARE_GC_Z_ZRELOCATE_HPP
diff --git a/src/hotspot/share/gc/z/zRelocationSet.cpp b/src/hotspot/share/gc/z/zRelocationSet.cpp
index 6577e11009c..7fab802a877 100644
--- a/src/hotspot/share/gc/z/zRelocationSet.cpp
+++ b/src/hotspot/share/gc/z/zRelocationSet.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,8 +23,12 @@
 
 #include "precompiled.hpp"
 #include "gc/z/zArray.inline.hpp"
+#include "gc/z/zCollectedHeap.hpp"
 #include "gc/z/zForwarding.inline.hpp"
 #include "gc/z/zForwardingAllocator.inline.hpp"
+#include "gc/z/zGeneration.inline.hpp"
+#include "gc/z/zPage.inline.hpp"
+#include "gc/z/zPageAllocator.hpp"
 #include "gc/z/zRelocationSet.inline.hpp"
 #include "gc/z/zRelocationSetSelector.inline.hpp"
 #include "gc/z/zStat.hpp"
@@ -38,35 +42,53 @@ private:
   ZForwardingAllocator* const    _allocator;
   ZForwarding**                  _forwardings;
   const size_t                   _nforwardings;
+  const ZArray<ZPage*>*          _small;
+  const ZArray<ZPage*>*          _medium;
   ZArrayParallelIterator<ZPage*> _small_iter;
   ZArrayParallelIterator<ZPage*> _medium_iter;
-  volatile size_t                _small_next;
-  volatile size_t                _medium_next;
 
-  void install(ZForwarding* forwarding, volatile size_t* next) {
-    const size_t index = Atomic::fetch_and_add(next, 1u);
+  void install(ZForwarding* forwarding, size_t index) {
     assert(index < _nforwardings, "Invalid index");
+
+    ZPage* const page = forwarding->page();
+
+    page->log_msg(" (relocation selected)");
+
     _forwardings[index] = forwarding;
+
+    if (forwarding->is_promotion()) {
+      // Before promoting an object (and before relocate start), we must ensure that all
+      // contained zpointers are store good. The marking code ensures that for non-null
+      // pointers, but null pointers are ignored. This code ensures that even null pointers
+      // are made store good, for the promoted objects.
+      page->object_iterate([&](oop obj) {
+        ZIterator::basic_oop_iterate_safe(obj, ZBarrier::promote_barrier_on_young_oop_field);
+      });
+    }
   }
 
-  void install_small(ZForwarding* forwarding) {
-    install(forwarding, &_small_next);
+  void install_small(ZForwarding* forwarding, size_t index) {
+    install(forwarding, index);
   }
 
-  void install_medium(ZForwarding* forwarding) {
-    install(forwarding, &_medium_next);
+  void install_medium(ZForwarding* forwarding, size_t index) {
+    install(forwarding, index);
+  }
+
+  ZPageAge to_age(ZPage* page) {
+    return ZRelocate::compute_to_age(page->age());
   }
 
 public:
   ZRelocationSetInstallTask(ZForwardingAllocator* allocator, const ZRelocationSetSelector* selector) :
       ZTask("ZRelocationSetInstallTask"),
       _allocator(allocator),
-      _forwardings(NULL),
-      _nforwardings(selector->small()->length() + selector->medium()->length()),
-      _small_iter(selector->small()),
-      _medium_iter(selector->medium()),
-      _small_next(selector->medium()->length()),
-      _medium_next(0) {
+      _forwardings(nullptr),
+      _nforwardings(selector->selected_small()->length() + selector->selected_medium()->length()),
+      _small(selector->selected_small()),
+      _medium(selector->selected_medium()),
+      _small_iter(selector->selected_small()),
+      _medium_iter(selector->selected_medium()) {
 
     // Reset the allocator to have room for the relocation
     // set, all forwardings, and all forwarding entries.
@@ -85,15 +107,17 @@ public:
 
   virtual void work() {
     // Allocate and install forwardings for small pages
-    for (ZPage* page; _small_iter.next(&page);) {
-      ZForwarding* const forwarding = ZForwarding::alloc(_allocator, page);
-      install_small(forwarding);
+    for (size_t page_index; _small_iter.next_index(&page_index);) {
+      ZPage* page = _small->at(int(page_index));
+      ZForwarding* const forwarding = ZForwarding::alloc(_allocator, page, to_age(page));
+      install_small(forwarding, _medium->length() + page_index);
     }
 
     // Allocate and install forwardings for medium pages
-    for (ZPage* page; _medium_iter.next(&page);) {
-      ZForwarding* const forwarding = ZForwarding::alloc(_allocator, page);
-      install_medium(forwarding);
+    for (size_t page_index; _medium_iter.next_index(&page_index);) {
+      ZPage* page = _medium->at(int(page_index));
+      ZForwarding* const forwarding = ZForwarding::alloc(_allocator, page, to_age(page));
+      install_medium(forwarding, page_index);
     }
   }
 
@@ -106,25 +130,49 @@ public:
   }
 };
 
-ZRelocationSet::ZRelocationSet(ZWorkers* workers) :
-    _workers(workers),
+ZRelocationSet::ZRelocationSet(ZGeneration* generation) :
+    _generation(generation),
     _allocator(),
-    _forwardings(NULL),
-    _nforwardings(0) {}
+    _forwardings(nullptr),
+    _nforwardings(0),
+    _promotion_lock(),
+    _flip_promoted_pages(),
+    _in_place_relocate_promoted_pages() {}
+
+ZWorkers* ZRelocationSet::workers() const {
+  return _generation->workers();
+}
+
+ZGeneration* ZRelocationSet::generation() const {
+  return _generation;
+}
+
+ZArray<ZPage*>* ZRelocationSet::flip_promoted_pages() {
+  return &_flip_promoted_pages;
+}
 
 void ZRelocationSet::install(const ZRelocationSetSelector* selector) {
   // Install relocation set
   ZRelocationSetInstallTask task(&_allocator, selector);
-  _workers->run(&task);
+  workers()->run(&task);
 
   _forwardings = task.forwardings();
   _nforwardings = task.nforwardings();
 
   // Update statistics
-  ZStatRelocation::set_at_install_relocation_set(_allocator.size());
+  _generation->stat_relocation()->at_install_relocation_set(_allocator.size());
 }
 
-void ZRelocationSet::reset() {
+static void destroy_and_clear(ZPageAllocator* page_allocator, ZArray<ZPage*>* array) {
+  for (int i = 0; i < array->length(); i++) {
+    // Delete non-relocating promoted pages from last cycle
+    ZPage* const page = array->at(i);
+    page_allocator->safe_destroy_page(page);
+  }
+
+  array->clear();
+}
+void ZRelocationSet::reset(ZPageAllocator* page_allocator) {
   // Destroy forwardings
   ZRelocationSetIterator iter(this);
   for (ZForwarding* forwarding; iter.next(&forwarding);) {
@@ -132,4 +180,21 @@ void ZRelocationSet::reset() {
   }
 
   _nforwardings = 0;
+
+  destroy_and_clear(page_allocator, &_in_place_relocate_promoted_pages);
+  destroy_and_clear(page_allocator, &_flip_promoted_pages);
+}
+
+void ZRelocationSet::register_flip_promoted(const ZArray<ZPage*>& pages) {
+  ZLocker<ZLock> locker(&_promotion_lock);
+  for (ZPage* const page : pages) {
+    assert(!_flip_promoted_pages.contains(page), "no duplicates allowed");
+    _flip_promoted_pages.append(page);
+  }
+}
+
+void ZRelocationSet::register_in_place_relocate_promoted(ZPage* page) {
+  ZLocker<ZLock> locker(&_promotion_lock);
+  assert(!_in_place_relocate_promoted_pages.contains(page), "no duplicates allowed");
+  _in_place_relocate_promoted_pages.append(page);
 }
diff --git a/src/hotspot/share/gc/z/zRelocationSet.hpp b/src/hotspot/share/gc/z/zRelocationSet.hpp
index 5881055db99..2052f3c7bf1 100644
--- a/src/hotspot/share/gc/z/zRelocationSet.hpp
+++ b/src/hotspot/share/gc/z/zRelocationSet.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,8 +26,12 @@
 
 #include "gc/z/zArray.hpp"
 #include "gc/z/zForwardingAllocator.hpp"
+#include "gc/z/zLock.hpp"
 
 class ZForwarding;
+class ZGeneration;
+class ZPage;
+class ZPageAllocator;
 class ZRelocationSetSelector;
 class ZWorkers;
 
@@ -35,16 +39,26 @@ class ZRelocationSet {
   template <bool> friend class ZRelocationSetIteratorImpl;
 
 private:
-  ZWorkers*            _workers;
+  ZGeneration*         _generation;
   ZForwardingAllocator _allocator;
   ZForwarding**        _forwardings;
   size_t               _nforwardings;
+  ZLock                _promotion_lock;
+  ZArray<ZPage*>       _flip_promoted_pages;
+  ZArray<ZPage*>       _in_place_relocate_promoted_pages;
+
+  ZWorkers* workers() const;
 
 public:
-  ZRelocationSet(ZWorkers* workers);
+  ZRelocationSet(ZGeneration* generation);
 
   void install(const ZRelocationSetSelector* selector);
-  void reset();
+  void reset(ZPageAllocator* page_allocator);
+  ZGeneration* generation() const;
+  ZArray<ZPage*>* flip_promoted_pages();
+
+  void register_flip_promoted(const ZArray<ZPage*>& pages);
+  void register_in_place_relocate_promoted(ZPage* page);
 };
 
 template <bool Parallel>
diff --git a/src/hotspot/share/gc/z/zRelocationSetSelector.cpp b/src/hotspot/share/gc/z/zRelocationSetSelector.cpp
index 04160480fe8..d5618e9207c 100644
--- a/src/hotspot/share/gc/z/zRelocationSetSelector.cpp
+++ b/src/hotspot/share/gc/z/zRelocationSetSelector.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,33 +34,37 @@
 #include "utilities/powerOfTwo.hpp"
 
 ZRelocationSetSelectorGroupStats::ZRelocationSetSelectorGroupStats() :
-    _npages(0),
+    _npages_candidates(0),
     _total(0),
     _live(0),
     _empty(0),
+    _npages_selected(0),
     _relocate(0) {}
 
 ZRelocationSetSelectorGroup::ZRelocationSetSelectorGroup(const char* name,
-                                                         uint8_t page_type,
+                                                         ZPageType page_type,
                                                          size_t page_size,
-                                                         size_t object_size_limit) :
+                                                         size_t object_size_limit,
+                                                         double fragmentation_limit) :
     _name(name),
     _page_type(page_type),
     _page_size(page_size),
     _object_size_limit(object_size_limit),
-    _fragmentation_limit(page_size * (ZFragmentationLimit / 100)),
+    _fragmentation_limit(fragmentation_limit),
+    _page_fragmentation_limit(page_size * (fragmentation_limit / 100)),
     _live_pages(),
+    _not_selected_pages(),
     _forwarding_entries(0),
     _stats() {}
 
 bool ZRelocationSetSelectorGroup::is_disabled() {
   // Medium pages are disabled when their page size is zero
-  return _page_type == ZPageTypeMedium && _page_size == 0;
+  return _page_type == ZPageType::medium && _page_size == 0;
 }
 
 bool ZRelocationSetSelectorGroup::is_selectable() {
   // Large pages are not selectable
-  return _page_type != ZPageTypeLarge;
+  return _page_type != ZPageType::large;
 }
 
 void ZRelocationSetSelectorGroup::semi_sort() {
@@ -90,14 +94,14 @@ void ZRelocationSetSelectorGroup::semi_sort() {
 
   // Allocate destination array
   const int npages = _live_pages.length();
-  ZArray<ZPage*> sorted_live_pages(npages, npages, NULL);
+  ZArray<ZPage*> sorted_live_pages(npages, npages, nullptr);
 
   // Sort pages into partitions
   ZArrayIterator<ZPage*> iter2(&_live_pages);
   for (ZPage* page; iter2.next(&page);) {
     const size_t index = page->live_bytes() >> partition_size_shift;
     const int finger = partitions[index]++;
-    assert(sorted_live_pages.at(finger) == NULL, "Invalid finger");
+    assert(sorted_live_pages.at(finger) == nullptr, "Invalid finger");
     sorted_live_pages.at_put(finger, page);
   }
 
@@ -111,8 +115,10 @@ void ZRelocationSetSelectorGroup::select_inner() {
   const int npages = _live_pages.length();
   int selected_from = 0;
   int selected_to = 0;
-  size_t selected_live_bytes = 0;
+  size_t npages_selected[ZPageAgeMax + 1] = { 0 };
+  size_t selected_live_bytes[ZPageAgeMax + 1] = { 0 };
   size_t selected_forwarding_entries = 0;
+
   size_t from_live_bytes = 0;
   size_t from_forwarding_entries = 0;
 
@@ -121,7 +127,8 @@ void ZRelocationSetSelectorGroup::select_inner() {
   for (int from = 1; from <= npages; from++) {
     // Add page to the candidate relocation set
     ZPage* const page = _live_pages.at(from - 1);
-    from_live_bytes += page->live_bytes();
+    const size_t page_live_bytes = page->live_bytes();
+    from_live_bytes += page_live_bytes;
     from_forwarding_entries += ZForwarding::nentries(page);
 
     // Calculate the maximum number of pages needed by the candidate relocation set.
@@ -137,27 +144,38 @@ void ZRelocationSetSelectorGroup::select_inner() {
     const int diff_from = from - selected_from;
     const int diff_to = to - selected_to;
     const double diff_reclaimable = 100 - percent_of(diff_to, diff_from);
-    if (diff_reclaimable > ZFragmentationLimit) {
+    if (diff_reclaimable > _fragmentation_limit) {
       selected_from = from;
       selected_to = to;
-      selected_live_bytes = from_live_bytes;
+      selected_live_bytes[static_cast<uint>(page->age())] += page_live_bytes;
+      npages_selected[static_cast<uint>(page->age())] += 1;
       selected_forwarding_entries = from_forwarding_entries;
     }
 
     log_trace(gc, reloc)("Candidate Relocation Set (%s Pages): %d->%d, "
-                         "%.1f%% relative defragmentation, " SIZE_FORMAT " forwarding entries, %s",
+                         "%.1f%% relative defragmentation, " SIZE_FORMAT " forwarding entries, %s, live %d",
                          _name, from, to, diff_reclaimable, from_forwarding_entries,
-                         (selected_from == from) ? "Selected" : "Rejected");
+                         (selected_from == from) ? "Selected" : "Rejected",
+                         int(page_live_bytes * 100 / page->size()));
   }
 
   // Finalize selection
+  for (int i = selected_from; i < _live_pages.length(); i++) {
+    ZPage* const page = _live_pages.at(i);
+    if (page->is_young()) {
+      _not_selected_pages.append(page);
+    }
+  }
   _live_pages.trunc_to(selected_from);
   _forwarding_entries = selected_forwarding_entries;
 
   // Update statistics
-  _stats._relocate = selected_live_bytes;
+  for (uint i = 0; i <= ZPageAgeMax; ++i) {
+    _stats[i]._relocate = selected_live_bytes[i];
+    _stats[i]._npages_selected = npages_selected[i];
+  }
 
-  log_trace(gc, reloc)("Relocation Set (%s Pages): %d->%d, %d skipped, " SIZE_FORMAT " forwarding entries",
+  log_debug(gc, reloc)("Relocation Set (%s Pages): %d->%d, %d skipped, " SIZE_FORMAT " forwarding entries",
                        _name, selected_from, selected_to, npages - selected_from, selected_forwarding_entries);
 }
 
@@ -170,16 +188,32 @@ void ZRelocationSetSelectorGroup::select() {
 
   if (is_selectable()) {
     select_inner();
+  } else {
+    // Mark pages as not selected
+    const int npages = _live_pages.length();
+    for (int from = 1; from <= npages; from++) {
+      ZPage* const page = _live_pages.at(from - 1);
+      _not_selected_pages.append(page);
+    }
+  }
+
+  ZRelocationSetSelectorGroupStats s{};
+  for (uint i = 0; i <= ZPageAgeMax; ++i) {
+    s._npages_candidates += _stats[i].npages_candidates();
+    s._total += _stats[i].total();
+    s._empty += _stats[i].empty();
+    s._npages_selected += _stats[i].npages_selected();
+    s._relocate += _stats[i].relocate();
   }
 
   // Send event
-  event.commit(_page_type, _stats.npages(), _stats.total(), _stats.empty(), _stats.relocate());
+  event.commit((u8)_page_type, s._npages_candidates, s._total, s._empty, s._npages_selected, s._relocate);
 }
 
-ZRelocationSetSelector::ZRelocationSetSelector() :
-    _small("Small", ZPageTypeSmall, ZPageSizeSmall, ZObjectSizeLimitSmall),
-    _medium("Medium", ZPageTypeMedium, ZPageSizeMedium, ZObjectSizeLimitMedium),
-    _large("Large", ZPageTypeLarge, 0 /* page_size */, 0 /* object_size_limit */),
+ZRelocationSetSelector::ZRelocationSetSelector(double fragmentation_limit) :
+    _small("Small", ZPageType::small, ZPageSizeSmall, ZObjectSizeLimitSmall, fragmentation_limit),
+    _medium("Medium", ZPageType::medium, ZPageSizeMedium, ZObjectSizeLimitMedium, fragmentation_limit),
+    _large("Large", ZPageType::large, 0 /* page_size */, 0 /* object_size_limit */, fragmentation_limit),
     _empty_pages() {}
 
 void ZRelocationSetSelector::select() {
@@ -202,8 +236,15 @@ void ZRelocationSetSelector::select() {
 
 ZRelocationSetSelectorStats ZRelocationSetSelector::stats() const {
   ZRelocationSetSelectorStats stats;
-  stats._small = _small.stats();
-  stats._medium = _medium.stats();
-  stats._large = _large.stats();
+
+  for (uint i = 0; i <= ZPageAgeMax; ++i) {
+    const ZPageAge age = static_cast<ZPageAge>(i);
+    stats._small[i] = _small.stats(age);
+    stats._medium[i] = _medium.stats(age);
+    stats._large[i] = _large.stats(age);
+  }
+
+  stats._has_relocatable_pages = total() > 0;
+
   return stats;
 }
diff --git a/src/hotspot/share/gc/z/zRelocationSetSelector.hpp b/src/hotspot/share/gc/z/zRelocationSetSelector.hpp
index 7a60234db90..7bf2942e843 100644
--- a/src/hotspot/share/gc/z/zRelocationSetSelector.hpp
+++ b/src/hotspot/share/gc/z/zRelocationSetSelector.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,9 @@
 #define SHARE_GC_Z_ZRELOCATIONSETSELECTOR_HPP
 
 #include "gc/z/zArray.hpp"
+#include "gc/z/zGenerationId.hpp"
+#include "gc/z/zPageAge.hpp"
+#include "gc/z/zPageType.hpp"
 #include "memory/allocation.hpp"
 
 class ZPage;
@@ -33,19 +36,25 @@ class ZRelocationSetSelectorGroupStats {
   friend class ZRelocationSetSelectorGroup;
 
 private:
-  size_t _npages;
+  // Candidate set
+  size_t _npages_candidates;
   size_t _total;
   size_t _live;
   size_t _empty;
+
+  // Selected set
+  size_t _npages_selected;
   size_t _relocate;
 
 public:
   ZRelocationSetSelectorGroupStats();
 
-  size_t npages() const;
+  size_t npages_candidates() const;
   size_t total() const;
   size_t live() const;
   size_t empty() const;
+
+  size_t npages_selected() const;
   size_t relocate() const;
 };
 
@@ -53,26 +62,32 @@ class ZRelocationSetSelectorStats {
   friend class ZRelocationSetSelector;
 
 private:
-  ZRelocationSetSelectorGroupStats _small;
-  ZRelocationSetSelectorGroupStats _medium;
-  ZRelocationSetSelectorGroupStats _large;
+  ZRelocationSetSelectorGroupStats _small[ZPageAgeMax + 1];
+  ZRelocationSetSelectorGroupStats _medium[ZPageAgeMax + 1];
+  ZRelocationSetSelectorGroupStats _large[ZPageAgeMax + 1];
+
+  size_t _has_relocatable_pages;
 
 public:
-  const ZRelocationSetSelectorGroupStats& small() const;
-  const ZRelocationSetSelectorGroupStats& medium() const;
-  const ZRelocationSetSelectorGroupStats& large() const;
+  const ZRelocationSetSelectorGroupStats& small(ZPageAge age) const;
+  const ZRelocationSetSelectorGroupStats& medium(ZPageAge age) const;
+  const ZRelocationSetSelectorGroupStats& large(ZPageAge age) const;
+
+  bool has_relocatable_pages() const;
 };
 
 class ZRelocationSetSelectorGroup {
 private:
   const char* const                _name;
-  const uint8_t                    _page_type;
+  const ZPageType                  _page_type;
   const size_t                     _page_size;
   const size_t                     _object_size_limit;
-  const size_t                     _fragmentation_limit;
+  const double                     _fragmentation_limit;
+  const size_t                     _page_fragmentation_limit;
   ZArray<ZPage*>                   _live_pages;
+  ZArray<ZPage*>                   _not_selected_pages;
   size_t                           _forwarding_entries;
-  ZRelocationSetSelectorGroupStats _stats;
+  ZRelocationSetSelectorGroupStats _stats[ZPageAgeMax + 1];
 
   bool is_disabled();
   bool is_selectable();
@@ -81,18 +96,21 @@ private:
 
 public:
   ZRelocationSetSelectorGroup(const char* name,
-                              uint8_t page_type,
+                              ZPageType page_type,
                               size_t page_size,
-                              size_t object_size_limit);
+                              size_t object_size_limit,
+                              double fragmentation_limit);
 
   void register_live_page(ZPage* page);
   void register_empty_page(ZPage* page);
   void select();
 
-  const ZArray<ZPage*>* selected() const;
+  const ZArray<ZPage*>* live_pages() const;
+  const ZArray<ZPage*>* selected_pages() const;
+  const ZArray<ZPage*>* not_selected_pages() const;
   size_t forwarding_entries() const;
 
-  const ZRelocationSetSelectorGroupStats& stats() const;
+  const ZRelocationSetSelectorGroupStats& stats(ZPageAge age) const;
 };
 
 class ZRelocationSetSelector : public StackObj {
@@ -107,7 +125,7 @@ private:
   size_t relocate() const;
 
 public:
-  ZRelocationSetSelector();
+  ZRelocationSetSelector(double fragmentation_limit);
 
   void register_live_page(ZPage* page);
   void register_empty_page(ZPage* page);
@@ -118,8 +136,12 @@ public:
 
   void select();
 
-  const ZArray<ZPage*>* small() const;
-  const ZArray<ZPage*>* medium() const;
+  const ZArray<ZPage*>* selected_small() const;
+  const ZArray<ZPage*>* selected_medium() const;
+
+  const ZArray<ZPage*>* not_selected_small() const;
+  const ZArray<ZPage*>* not_selected_medium() const;
+  const ZArray<ZPage*>* not_selected_large() const;
   size_t forwarding_entries() const;
 
   ZRelocationSetSelectorStats stats() const;
diff --git a/src/hotspot/share/gc/z/zRelocationSetSelector.inline.hpp b/src/hotspot/share/gc/z/zRelocationSetSelector.inline.hpp
index 2f31e108856..316409ac7d8 100644
--- a/src/hotspot/share/gc/z/zRelocationSetSelector.inline.hpp
+++ b/src/hotspot/share/gc/z/zRelocationSetSelector.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,8 +29,8 @@
 #include "gc/z/zArray.inline.hpp"
 #include "gc/z/zPage.inline.hpp"
 
-inline size_t ZRelocationSetSelectorGroupStats::npages() const {
-  return _npages;
+inline size_t ZRelocationSetSelectorGroupStats::npages_candidates() const {
+  return _npages_candidates;
 }
 
 inline size_t ZRelocationSetSelectorGroupStats::total() const {
@@ -45,63 +45,81 @@ inline size_t ZRelocationSetSelectorGroupStats::empty() const {
   return _empty;
 }
 
+inline size_t ZRelocationSetSelectorGroupStats::npages_selected() const {
+  return _npages_selected;
+}
+
 inline size_t ZRelocationSetSelectorGroupStats::relocate() const {
   return _relocate;
 }
 
-inline const ZRelocationSetSelectorGroupStats& ZRelocationSetSelectorStats::small() const {
-  return _small;
+inline bool ZRelocationSetSelectorStats::has_relocatable_pages() const {
+  return _has_relocatable_pages;
 }
 
-inline const ZRelocationSetSelectorGroupStats& ZRelocationSetSelectorStats::medium() const {
-  return _medium;
+inline const ZRelocationSetSelectorGroupStats& ZRelocationSetSelectorStats::small(ZPageAge age) const {
+  return _small[static_cast<uint>(age)];
 }
 
-inline const ZRelocationSetSelectorGroupStats& ZRelocationSetSelectorStats::large() const {
-  return _large;
+inline const ZRelocationSetSelectorGroupStats& ZRelocationSetSelectorStats::medium(ZPageAge age) const {
+  return _medium[static_cast<uint>(age)];
+}
+
+inline const ZRelocationSetSelectorGroupStats& ZRelocationSetSelectorStats::large(ZPageAge age) const {
+  return _large[static_cast<uint>(age)];
 }
 
 inline void ZRelocationSetSelectorGroup::register_live_page(ZPage* page) {
-  const uint8_t type = page->type();
   const size_t size = page->size();
   const size_t live = page->live_bytes();
   const size_t garbage = size - live;
 
-  if (garbage > _fragmentation_limit) {
+  // Pre-filter out pages that are guaranteed to not be selected
+  if (!page->is_large() && garbage > _page_fragmentation_limit) {
     _live_pages.append(page);
+  } else if (page->is_young()) {
+    _not_selected_pages.append(page);
   }
 
-  _stats._npages++;
-  _stats._total += size;
-  _stats._live += live;
+  const uint age = static_cast<uint>(page->age());
+  _stats[age]._npages_candidates++;
+  _stats[age]._total += size;
+  _stats[age]._live += live;
 }
 
 inline void ZRelocationSetSelectorGroup::register_empty_page(ZPage* page) {
   const size_t size = page->size();
 
-  _stats._npages++;
-  _stats._total += size;
-  _stats._empty += size;
+  const uint age = static_cast<uint>(page->age());
+  _stats[age]._npages_candidates++;
+  _stats[age]._total += size;
+  _stats[age]._empty += size;
 }
 
-inline const ZArray<ZPage*>* ZRelocationSetSelectorGroup::selected() const {
+inline const ZArray<ZPage*>* ZRelocationSetSelectorGroup::selected_pages() const {
   return &_live_pages;
 }
 
+inline const ZArray<ZPage*>* ZRelocationSetSelectorGroup::not_selected_pages() const {
+  return &_not_selected_pages;
+}
+
 inline size_t ZRelocationSetSelectorGroup::forwarding_entries() const {
   return _forwarding_entries;
 }
 
-inline const ZRelocationSetSelectorGroupStats& ZRelocationSetSelectorGroup::stats() const {
-  return _stats;
+inline const ZRelocationSetSelectorGroupStats& ZRelocationSetSelectorGroup::stats(ZPageAge age) const {
+  return _stats[static_cast<uint>(age)];
 }
 
 inline void ZRelocationSetSelector::register_live_page(ZPage* page) {
-  const uint8_t type = page->type();
+  page->log_msg(" (relocation candidate)");
 
-  if (type == ZPageTypeSmall) {
+  const ZPageType type = page->type();
+
+  if (type == ZPageType::small) {
     _small.register_live_page(page);
-  } else if (type == ZPageTypeMedium) {
+  } else if (type == ZPageType::medium) {
     _medium.register_live_page(page);
   } else {
     _large.register_live_page(page);
@@ -109,11 +127,13 @@ inline void ZRelocationSetSelector::register_live_page(ZPage* page) {
 }
 
 inline void ZRelocationSetSelector::register_empty_page(ZPage* page) {
-  const uint8_t type = page->type();
+  page->log_msg(" (relocation empty)");
 
-  if (type == ZPageTypeSmall) {
+  const ZPageType type = page->type();
+
+  if (type == ZPageType::small) {
     _small.register_empty_page(page);
-  } else if (type == ZPageTypeMedium) {
+  } else if (type == ZPageType::medium) {
     _medium.register_empty_page(page);
   } else {
     _large.register_empty_page(page);
@@ -135,23 +155,50 @@ inline void ZRelocationSetSelector::clear_empty_pages() {
 }
 
 inline size_t ZRelocationSetSelector::total() const {
-  return _small.stats().total() + _medium.stats().total() + _large.stats().total();
+  size_t sum = 0;
+  for (uint i = 0; i <= ZPageAgeMax; ++i) {
+    const ZPageAge age = static_cast<ZPageAge>(i);
+    sum += _small.stats(age).total() + _medium.stats(age).total() + _large.stats(age).total();
+  }
+  return sum;
 }
 
 inline size_t ZRelocationSetSelector::empty() const {
-  return _small.stats().empty() + _medium.stats().empty() + _large.stats().empty();
+  size_t sum = 0;
+  for (uint i = 0; i <= ZPageAgeMax; ++i) {
+    const ZPageAge age = static_cast<ZPageAge>(i);
+    sum += _small.stats(age).empty() + _medium.stats(age).empty() + _large.stats(age).empty();
+  }
+  return sum;
 }
 
 inline size_t ZRelocationSetSelector::relocate() const {
-  return _small.stats().relocate() + _medium.stats().relocate() + _large.stats().relocate();
+  size_t sum = 0;
+  for (uint i = 0; i <= ZPageAgeMax; ++i) {
+    const ZPageAge age = static_cast<ZPageAge>(i);
+    sum += _small.stats(age).relocate() + _medium.stats(age).relocate() + _large.stats(age).relocate();
+  }
+  return sum;
 }
 
-inline const ZArray<ZPage*>* ZRelocationSetSelector::small() const {
-  return _small.selected();
+inline const ZArray<ZPage*>* ZRelocationSetSelector::selected_small() const {
+  return _small.selected_pages();
 }
 
-inline const ZArray<ZPage*>* ZRelocationSetSelector::medium() const {
-  return _medium.selected();
+inline const ZArray<ZPage*>* ZRelocationSetSelector::selected_medium() const {
+  return _medium.selected_pages();
+}
+
+inline const ZArray<ZPage*>* ZRelocationSetSelector::not_selected_small() const {
+  return _small.not_selected_pages();
+}
+
+inline const ZArray<ZPage*>* ZRelocationSetSelector::not_selected_medium() const {
+  return _medium.not_selected_pages();
+}
+
+inline const ZArray<ZPage*>* ZRelocationSetSelector::not_selected_large() const {
+  return _large.not_selected_pages();
 }
 
 inline size_t ZRelocationSetSelector::forwarding_entries() const {
diff --git a/src/hotspot/share/gc/z/zRemembered.cpp b/src/hotspot/share/gc/z/zRemembered.cpp
new file mode 100644
index 00000000000..95865a5203f
--- /dev/null
+++ b/src/hotspot/share/gc/z/zRemembered.cpp
@@ -0,0 +1,591 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zForwarding.inline.hpp"
+#include "gc/z/zGeneration.inline.hpp"
+#include "gc/z/zHeap.inline.hpp"
+#include "gc/z/zIterator.inline.hpp"
+#include "gc/z/zMark.hpp"
+#include "gc/z/zPage.inline.hpp"
+#include "gc/z/zPageTable.hpp"
+#include "gc/z/zRemembered.inline.hpp"
+#include "gc/z/zRememberedSet.hpp"
+#include "gc/z/zTask.hpp"
+#include "gc/z/zVerify.hpp"
+#include "memory/iterator.hpp"
+#include "oops/oop.inline.hpp"
+#include "utilities/bitMap.inline.hpp"
+#include "utilities/debug.hpp"
+
+ZRemembered::ZRemembered(ZPageTable* page_table,
+                         const ZForwardingTable* old_forwarding_table,
+                         ZPageAllocator* page_allocator) :
+    _page_table(page_table),
+    _old_forwarding_table(old_forwarding_table),
+    _page_allocator(page_allocator),
+    _found_old() {
+}
+
+template <typename Function>
+void ZRemembered::oops_do_forwarded_via_containing(GrowableArrayView<ZRememberedSetContaining>* array, Function function) const {
+  // The array contains duplicated from_addr values. Cache expensive operations.
+  zaddress_unsafe from_addr = zaddress_unsafe::null;
+  zaddress to_addr = zaddress::null;
+  size_t object_size = 0;
+
+  for (const ZRememberedSetContaining containing: *array) {
+    if (from_addr != containing._addr) {
+      from_addr = containing._addr;
+
+      // Relocate object to new location
+      to_addr = ZGeneration::old()->relocate_or_remap_object(from_addr);
+
+      // Figure out size
+      object_size = ZUtils::object_size(to_addr);
+    }
+
+    // Calculate how far into the from-object the remset entry is
+    const uintptr_t field_offset = containing._field_addr - from_addr;
+
+    // The 'containing' could contain mismatched (addr, addr_field).
+    // Need to check if the field was within the reported object.
+    if (field_offset < object_size) {
+      // Calculate the corresponding address in the to-object
+      const zaddress to_addr_field = to_addr + field_offset;
+
+      function((volatile zpointer*)untype(to_addr_field));
+    }
+  }
+}
+
+bool ZRemembered::should_scan_page(ZPage* page) const {
+  if (!ZGeneration::old()->is_phase_relocate()) {
+    // If the old generation collection is not in the relocation phase, then it
+    // will not need any synchronization on its forwardings.
+    return true;
+  }
+
+  ZForwarding* const forwarding = ZGeneration::old()->forwarding(ZOffset::address_unsafe(page->start()));
+
+  if (forwarding == nullptr) {
+    // This page was provably not part of the old relocation set
+    return true;
+  }
+
+  if (!forwarding->relocated_remembered_fields_is_concurrently_scanned()) {
+    // Safe to scan
+    return true;
+  }
+
+  // If we get here, we know that the old collection is concurrently relocating
+  // objects. We need to be extremely careful not to scan a page that is
+  // concurrently being in-place relocated because it's objects and previous
+  // bits could be concurrently be moving around.
+  //
+  // Before calling this function ZRemembered::scan_forwarding ensures
+  // that all forwardings that have not already been fully relocated,
+  // will have had their "previous" remembered set bits scanned.
+  //
+  // The current page we're currently scanning could either be the same page
+  // that was found during scan_forwarding, or it could have been replaced
+  // by a new "allocating" page. There are two situations we have to consider:
+  //
+  // 1) If it is a proper new allocating page, then all objects where copied
+  // after scan_forwarding ran, and we are guaranteed that no "previous"
+  // remembered set bits are set. So, there's no need to scan this page.
+  //
+  // 2) If this is an in-place relocated page, then the entire page could
+  // be concurrently relocated. Meaning that both objects and previous
+  // remembered set bits could be moving around. However, if the in-place
+  // relocation is ongoing, we've already scanned all relevant "previous"
+  // bits when calling scan_forwarding. So, this page *must* not be scanned.
+  //
+  // Don't scan the page.
+  return false;
+}
+
+bool ZRemembered::scan_page(ZPage* page) const {
+  const bool can_trust_live_bits =
+      page->is_relocatable() && !ZGeneration::old()->is_phase_mark();
+
+  bool result = false;
+
+  if (!can_trust_live_bits) {
+    // We don't have full liveness info - scan all remset entries
+    page->log_msg(" (scan_page_remembered)");
+    int count = 0;
+    page->oops_do_remembered([&](volatile zpointer* p) {
+      result |= scan_field(p);
+      count++;
+    });
+    page->log_msg(" (scan_page_remembered done: %d ignoring: " PTR_FORMAT " )", count, p2i(page->remset_current()));
+  } else if (page->is_marked()) {
+    // We have full liveness info - Only scan remset entries in live objects
+    page->log_msg(" (scan_page_remembered_in_live)");
+    page->oops_do_remembered_in_live([&](volatile zpointer* p) {
+      result |= scan_field(p);
+    });
+  } else {
+    page->log_msg(" (scan_page_remembered_dead)");
+    // All objects are dead - do nothing
+  }
+
+  return result;
+}
+
+static void fill_containing(GrowableArrayCHeap<ZRememberedSetContaining, mtGC>* array, ZPage* page) {
+  page->log_msg(" (fill_remembered_containing)");
+
+  ZRememberedSetContainingIterator iter(page);
+
+  for (ZRememberedSetContaining containing; iter.next(&containing);) {
+    array->push(containing);
+  }
+}
+
+struct ZRememberedScanForwardingContext {
+  GrowableArrayCHeap<ZRememberedSetContaining, mtGC> _containing_array;
+
+  struct Where {
+    static const int NumRecords = 10;
+
+    Tickspan _duration;
+    int      _count;
+    Tickspan _max_durations[NumRecords];
+    int      _max_count;
+
+    Where() :
+        _duration(),
+        _count(),
+        _max_durations(),
+        _max_count() {}
+
+    void report(const Tickspan& duration) {
+      _duration += duration;
+      _count++;
+
+      // Install into max array
+      for (int i = 0; i < NumRecords; i++) {
+        if (duration > _max_durations[i]) {
+          // Slid to the side
+          for (int j = _max_count - 1; i < j; j--) {
+            _max_durations[j] = _max_durations[j - 1];
+          }
+
+          // Install
+          _max_durations[i] = duration;
+          if (_max_count < NumRecords) {
+            _max_count++;
+          }
+          break;
+        }
+      }
+    }
+
+    void print(const char* name) {
+      log_debug(gc, remset)("Remset forwarding %s: %.3fms count: %d %s",
+          name, TimeHelper::counter_to_millis(_duration.value()), _count, Thread::current()->name());
+      for (int i = 0; i < _max_count; i++) {
+        log_debug(gc, remset)("  %.3fms", TimeHelper::counter_to_millis(_max_durations[i].value()));
+      }
+    }
+  };
+
+  Where _where[2];
+
+  ZRememberedScanForwardingContext() :
+      _containing_array(),
+      _where() {}
+
+  ~ZRememberedScanForwardingContext() {
+    print();
+  }
+
+  void report_retained(const Tickspan& duration) {
+    _where[0].report(duration);
+  }
+
+  void report_released(const Tickspan& duration) {
+    _where[1].report(duration);
+  }
+
+  void print() {
+    _where[0].print("retained");
+    _where[1].print("released");
+  }
+};
+
+struct ZRememberedScanForwardingMeasureRetained {
+  ZRememberedScanForwardingContext* _context;
+  Ticks                             _start;
+
+  ZRememberedScanForwardingMeasureRetained(ZRememberedScanForwardingContext* context) :
+      _context(context),
+      _start(Ticks::now()) {
+  }
+
+  ~ZRememberedScanForwardingMeasureRetained() {
+    const Ticks end = Ticks::now();
+    const Tickspan duration = end - _start;
+    _context->report_retained(duration);
+  }
+};
+
+struct ZRememberedScanForwardingMeasureReleased {
+  ZRememberedScanForwardingContext* _context;
+  Ticks                             _start;
+
+  ZRememberedScanForwardingMeasureReleased(ZRememberedScanForwardingContext* context) :
+      _context(context),
+      _start(Ticks::now()) {
+  }
+
+  ~ZRememberedScanForwardingMeasureReleased() {
+    const Ticks end = Ticks::now();
+    const Tickspan duration = end - _start;
+    _context->report_released(duration);
+  }
+};
+
+bool ZRemembered::scan_forwarding(ZForwarding* forwarding, void* context_void) const {
+  ZRememberedScanForwardingContext* const context = (ZRememberedScanForwardingContext*)context_void;
+  bool result = false;
+
+  if (forwarding->retain_page(ZGeneration::old()->relocate_queue())) {
+    ZRememberedScanForwardingMeasureRetained measure(context);
+    forwarding->page()->log_msg(" (scan_forwarding)");
+
+    // We don't want to wait for the old relocation to finish and publish all
+    // relocated remembered fields. Reject its fields and collect enough data
+    // up-front.
+    forwarding->relocated_remembered_fields_notify_concurrent_scan_of();
+
+    // Collect all remset info while the page is retained
+    GrowableArrayCHeap<ZRememberedSetContaining, mtGC>* array = &context->_containing_array;
+    array->clear();
+    fill_containing(array, forwarding->page());
+    forwarding->release_page();
+
+    // Relocate (and mark) while page is released, to prevent
+    // retain deadlock when relocation threads in-place relocate.
+    oops_do_forwarded_via_containing(array, [&](volatile zpointer* p) {
+      result |= scan_field(p);
+    });
+
+  } else {
+    ZRememberedScanForwardingMeasureReleased measure(context);
+
+    // The page has been released. If the page was relocated while this young
+    // generation collection was running, the old generation relocation will
+    // have published all addresses of fields that had a remembered set entry.
+    forwarding->relocated_remembered_fields_apply_to_published([&](volatile zpointer* p) {
+      result |= scan_field(p);
+    });
+  }
+
+  return result;
+}
+
+// When scanning the remembered set during the young generation marking, we
+// want to visit all old pages. And we want that to be done in parallel and
+// fast.
+//
+// Walking over the entire page table and letting the workers claim indices
+// have been shown to have scalability issues.
+//
+// So, we have the "found old" optimization, which allows us to perform much
+// fewer claims (order of old pages, instead of order of slots in the page
+// table), and it allows us to read fewer pages.
+//
+// The set of "found old pages" isn't precise, and can contain stale entries
+// referring to slots of freed pages, or even slots where young pages have
+// been installed. However, it will not lack any of the old pages.
+//
+// The data is maintained very similar to when and how we maintain the
+// remembered set bits: We keep two separates sets, one for read-only access
+// by the young marking, and a currently active set where we register new
+// pages. When pages get relocated, or die, the page table slot for that page
+// must be cleared. This clearing is done just like we do with the remset
+// scanning: The old entries are not copied to the current active set, only
+// slots that were found to actually contain old pages are registered in the
+// active set.
+
+ZRemembered::FoundOld::FoundOld() :
+    // Array initialization requires copy constructors, which CHeapBitMap
+    // doesn't provide. Instantiate two instances, and populate an array
+    // with pointers to the two instances.
+    _allocated_bitmap_0{ZAddressOffsetMax >> ZGranuleSizeShift, mtGC, true /* clear */},
+    _allocated_bitmap_1{ZAddressOffsetMax >> ZGranuleSizeShift, mtGC, true /* clear */},
+    _bitmaps{&_allocated_bitmap_0, &_allocated_bitmap_1},
+    _current{0} {}
+
+BitMap* ZRemembered::FoundOld::current_bitmap() {
+  return _bitmaps[_current];
+}
+
+BitMap* ZRemembered::FoundOld::previous_bitmap() {
+  return _bitmaps[_current ^ 1];
+}
+
+void ZRemembered::FoundOld::flip() {
+  _current ^= 1;
+}
+
+void ZRemembered::FoundOld::clear_previous() {
+  previous_bitmap()->clear_large();
+}
+
+void ZRemembered::FoundOld::register_page(ZPage* page) {
+  assert(page->is_old(), "Only register old pages");
+  current_bitmap()->par_set_bit(untype(page->start()) >> ZGranuleSizeShift, memory_order_relaxed);
+}
+
+void ZRemembered::flip_found_old_sets() {
+  _found_old.flip();
+}
+
+void ZRemembered::clear_found_old_previous_set() {
+  _found_old.clear_previous();
+}
+
+void ZRemembered::register_found_old(ZPage* page) {
+  assert(page->is_old(), "Should only register old pages");
+  _found_old.register_page(page);
+}
+
+struct ZRemsetTableEntry {
+  ZPage* _page;
+  ZForwarding* _forwarding;
+};
+
+class ZRemsetTableIterator {
+private:
+  ZRemembered* const            _remembered;
+  ZPageTable* const             _page_table;
+  const ZForwardingTable* const _old_forwarding_table;
+  volatile BitMap::idx_t        _claimed;
+
+public:
+  ZRemsetTableIterator(ZRemembered* remembered) :
+      _remembered(remembered),
+      _page_table(remembered->_page_table),
+      _old_forwarding_table(remembered->_old_forwarding_table),
+      _claimed(0) {}
+
+  // This iterator uses the "found old" optimization.
+  bool next(ZRemsetTableEntry* entry_addr)  {
+    BitMap* const bm = _remembered->_found_old.previous_bitmap();
+
+    BitMap::idx_t prev = Atomic::load(&_claimed);
+
+    for (;;) {
+      if (prev == bm->size()) {
+        return false;
+      }
+
+      const BitMap::idx_t page_index = bm->find_first_set_bit(_claimed);
+      if (page_index == bm->size()) {
+        Atomic::cmpxchg(&_claimed, prev, page_index, memory_order_relaxed);
+        return false;
+      }
+
+      const BitMap::idx_t res = Atomic::cmpxchg(&_claimed, prev, page_index + 1, memory_order_relaxed);
+      if (res != prev) {
+        // Someone else claimed
+        prev = res;
+        continue;
+      }
+
+      // Found bit - look around for page or forwarding to scan
+
+      ZForwarding* forwarding = nullptr;
+      if (ZGeneration::old()->is_phase_relocate()) {
+        forwarding = _old_forwarding_table->at(page_index);
+      }
+
+      ZPage* page = _page_table->at(page_index);
+      if (page != nullptr && !page->is_old()) {
+        page = nullptr;
+      }
+
+      if (page == nullptr && forwarding == nullptr) {
+        // Nothing to scan
+        continue;
+      }
+
+      // Found old page or old forwarding
+      entry_addr->_forwarding = forwarding;
+      entry_addr->_page = page;
+
+      return true;
+    }
+  }
+};
+
+// This task scans the remembered set and follows pointers when possible.
+// Interleaving remembered set scanning with marking makes the marking times
+// lower and more predictable.
+class ZRememberedScanMarkFollowTask : public ZRestartableTask {
+private:
+  ZRemembered* const   _remembered;
+  ZMark* const         _mark;
+  ZRemsetTableIterator _remset_table_iterator;
+
+public:
+  ZRememberedScanMarkFollowTask(ZRemembered* remembered, ZMark* mark) :
+      ZRestartableTask("ZRememberedScanMarkFollowTask"),
+      _remembered(remembered),
+      _mark(mark),
+      _remset_table_iterator(remembered)  {
+    _mark->prepare_work();
+    _remembered->_page_allocator->enable_safe_destroy();
+    _remembered->_page_allocator->enable_safe_recycle();
+  }
+
+  ~ZRememberedScanMarkFollowTask() {
+    _remembered->_page_allocator->disable_safe_recycle();
+    _remembered->_page_allocator->disable_safe_destroy();
+    _mark->finish_work();
+    // We are done scanning the set of old pages.
+    // Clear the set for the next young collection.
+    _remembered->clear_found_old_previous_set();
+  }
+
+  virtual void work_inner() {
+    ZRememberedScanForwardingContext context;
+
+    // Follow initial roots
+    if (!_mark->follow_work_partial()) {
+      // Bail
+      return;
+    }
+
+    for (ZRemsetTableEntry entry; _remset_table_iterator.next(&entry);) {
+      bool left_marking = false;
+      ZForwarding* forwarding = entry._forwarding;
+      ZPage* page = entry._page;
+
+      // Scan forwarding
+      if (forwarding != nullptr) {
+        bool found_roots = _remembered->scan_forwarding(forwarding, &context);
+        ZVerify::after_scan(forwarding);
+        if (found_roots) {
+          // Follow remembered set when possible
+          left_marking = !_mark->follow_work_partial();
+        }
+      }
+
+      // Scan page
+      if (page != nullptr) {
+        if (_remembered->should_scan_page(page)) {
+          // Visit all entries pointing into young gen
+          bool found_roots = _remembered->scan_page(page);
+
+          // ... and as a side-effect clear the previous entries
+          if (ZVerifyRemembered) {
+            // Make sure self healing of pointers is ordered before clearing of
+            // the previous bits so that ZVerify::after_scan can detect missing
+            // remset entries accurately.
+            OrderAccess::storestore();
+          }
+          page->clear_remset_previous();
+
+          if (found_roots && !left_marking) {
+            // Follow remembered set when possible
+            left_marking = !_mark->follow_work_partial();
+          }
+        }
+
+        // The remset scanning maintains the "maybe old" pages optimization.
+        //
+        // We maintain two sets of old pages: The first is the currently active
+        // set, where old pages are registered into. The second is the old
+        // read-only copy. The two sets flip during young mark start. This
+        // analogous to how we set and clean remembered set bits.
+        //
+        // The iterator reads from the read-only copy, and then here, we install
+        // entries in the current active set.
+        _remembered->register_found_old(page);
+      }
+
+      SuspendibleThreadSet::yield();
+      if (left_marking) {
+        // Bail
+        return;
+      }
+    }
+
+    _mark->follow_work_complete();
+  }
+
+  virtual void work() {
+    SuspendibleThreadSetJoiner sts_joiner;
+    work_inner();
+    // We might have found pointers into the other generation, and then we want to
+    // publish such marking stacks to prevent that generation from getting a mark continue.
+    // We also flush in case of a resize where a new worker thread continues the marking
+    // work, causing a mark continue for the collected generation.
+    ZHeap::heap()->mark_flush_and_free(Thread::current());
+  }
+
+  virtual void resize_workers(uint nworkers) {
+    _mark->resize_workers(nworkers);
+  }
+};
+
+void ZRemembered::scan_and_follow(ZMark* mark) {
+  {
+    // Follow the object graph and lazily scan the remembered set
+    ZRememberedScanMarkFollowTask task(this, mark);
+    ZGeneration::young()->workers()->run(&task);
+
+    // Try to terminate after following the graph
+    if (ZAbort::should_abort() || !mark->try_terminate_flush()) {
+      return;
+    }
+  }
+
+  // If flushing failed, we have to restart marking again, but this time we don't need to
+  // scan the remembered set.
+  mark->mark_follow();
+}
+
+bool ZRemembered::scan_field(volatile zpointer* p) const {
+  assert(ZGeneration::young()->is_phase_mark(), "Wrong phase");
+
+  const zaddress addr = ZBarrier::remset_barrier_on_oop_field(p);
+
+  if (!is_null(addr) && ZHeap::heap()->is_young(addr)) {
+    remember(p);
+    return true;
+  }
+
+  return false;
+}
+
+void ZRemembered::flip() {
+  ZRememberedSet::flip();
+  flip_found_old_sets();
+}
diff --git a/src/hotspot/share/gc/z/zRemembered.hpp b/src/hotspot/share/gc/z/zRemembered.hpp
new file mode 100644
index 00000000000..f7c74e8c531
--- /dev/null
+++ b/src/hotspot/share/gc/z/zRemembered.hpp
@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZREMEMBERED_HPP
+#define SHARE_GC_Z_ZREMEMBERED_HPP
+
+#include "gc/z/zAddress.hpp"
+#include "utilities/bitMap.hpp"
+
+template <typename T> class GrowableArrayView;
+class OopClosure;
+class ZForwarding;
+class ZForwardingTable;
+class ZMark;
+class ZPage;
+class ZPageAllocator;
+class ZPageTable;
+struct ZRememberedSetContaining;
+
+class ZRemembered {
+  friend class ZRememberedScanMarkFollowTask;
+  friend class ZRemsetTableIterator;
+
+private:
+  ZPageTable* const             _page_table;
+  const ZForwardingTable* const _old_forwarding_table;
+  ZPageAllocator* const         _page_allocator;
+
+  // Optimization aid for faster old pages iteration
+  struct FoundOld {
+    CHeapBitMap   _allocated_bitmap_0;
+    CHeapBitMap   _allocated_bitmap_1;
+    BitMap* const _bitmaps[2];
+    int           _current;
+
+    FoundOld();
+
+    void flip();
+    void clear_previous();
+
+    void register_page(ZPage* page);
+
+    BitMap* current_bitmap();
+    BitMap* previous_bitmap();
+  } _found_old;
+
+  // Old pages iteration optimization aid
+  void flip_found_old_sets();
+  void clear_found_old_previous_set();
+
+  template <typename Function>
+  void oops_do_forwarded_via_containing(GrowableArrayView<ZRememberedSetContaining>* array, Function function) const;
+
+  bool should_scan_page(ZPage* page) const;
+
+  bool scan_page(ZPage* page) const;
+  bool scan_forwarding(ZForwarding* forwarding, void* context) const;
+
+public:
+  ZRemembered(ZPageTable* page_table,
+              const ZForwardingTable* old_forwarding_table,
+              ZPageAllocator* page_allocator);
+
+  // Add to remembered set
+  void remember(volatile zpointer* p) const;
+
+  // Scan all remembered sets and follow
+  void scan_and_follow(ZMark* mark);
+
+  // Save the current remembered sets,
+  // and switch over to empty remembered sets.
+  void flip();
+
+  // Scan a remembered set entry
+  bool scan_field(volatile zpointer* p) const;
+
+  // Verification
+  bool is_remembered(volatile zpointer* p) const;
+
+  // Register pages with the remembered set
+  void register_found_old(ZPage* page);
+};
+
+#endif // SHARE_GC_Z_ZREMEMBERED_HPP
diff --git a/src/hotspot/share/gc/z/zRemembered.inline.hpp b/src/hotspot/share/gc/z/zRemembered.inline.hpp
new file mode 100644
index 00000000000..8b6cb24de03
--- /dev/null
+++ b/src/hotspot/share/gc/z/zRemembered.inline.hpp
@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZREMEMBERED_INLINE_HPP
+#define SHARE_GC_Z_ZREMEMBERED_INLINE_HPP
+
+#include "gc/z/zRemembered.hpp"
+
+#include "gc/z/zBarrier.inline.hpp"
+#include "gc/z/zHeap.inline.hpp"
+#include "gc/z/zPage.inline.hpp"
+#include "gc/z/zPageTable.inline.hpp"
+
+inline void ZRemembered::remember(volatile zpointer* p) const {
+  ZPage* page = _page_table->get(p);
+  assert(page != nullptr,  "Page missing in page table");
+  page->remember(p);
+}
+
+inline bool ZRemembered::is_remembered(volatile zpointer* p) const {
+  ZPage* page = _page_table->get(p);
+  assert(page != nullptr,  "Page missing in page table");
+  return page->is_remembered(p);
+}
+
+#endif // SHARE_GC_Z_ZREMEMBERED_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zRememberedSet.cpp b/src/hotspot/share/gc/z/zRememberedSet.cpp
new file mode 100644
index 00000000000..7c0ab9ad6cf
--- /dev/null
+++ b/src/hotspot/share/gc/z/zRememberedSet.cpp
@@ -0,0 +1,227 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zBitMap.inline.hpp"
+#include "gc/z/zHeap.inline.hpp"
+#include "gc/z/zPage.inline.hpp"
+#include "gc/z/zRememberedSet.hpp"
+#include "logging/log.hpp"
+#include "memory/allocation.hpp"
+#include "memory/iterator.hpp"
+#include "utilities/globalDefinitions.hpp"
+
+int ZRememberedSet::_current = 0;
+
+void ZRememberedSet::flip() {
+  _current ^= 1;
+}
+
+ZRememberedSet::ZRememberedSet() :
+    _bitmap{ZMovableBitMap(), ZMovableBitMap()} {
+  // Defer initialization of the bitmaps until the owning
+  // page becomes old and its remembered set is initialized.
+}
+
+bool ZRememberedSet::is_initialized() const {
+  return _bitmap[0].size() > 0;
+}
+
+void ZRememberedSet::initialize(size_t page_size) {
+  assert(!is_initialized(), "precondition");
+  const BitMap::idx_t size_in_bits = to_bit_size(page_size);
+  _bitmap[0].initialize(size_in_bits, true /* clear */);
+  _bitmap[1].initialize(size_in_bits, true /* clear */);
+}
+
+void ZRememberedSet::resize(size_t page_size) {
+  // The bitmaps only need to be resized if remset has been
+  // initialized, and hence the bitmaps have been initialized.
+  if (is_initialized()) {
+    const BitMap::idx_t size_in_bits = to_bit_size(page_size);
+
+    // The bitmaps need to be cleared when free, but since this function is
+    // only used for shrinking the clear argument is correct but not crucial.
+    assert(size_in_bits <= _bitmap[0].size(), "Only used for shrinking");
+    _bitmap[0].resize(size_in_bits, true /* clear */);
+    _bitmap[1].resize(size_in_bits, true /* clear */);
+  }
+}
+
+bool ZRememberedSet::is_cleared_current() const {
+  return current()->is_empty();
+}
+
+bool ZRememberedSet::is_cleared_previous() const {
+  return previous()->is_empty();
+}
+
+void ZRememberedSet::clear_all() {
+  clear_current();
+  clear_previous();
+}
+
+void ZRememberedSet::clear_current() {
+  current()->clear_large();
+}
+
+void ZRememberedSet::clear_previous() {
+  previous()->clear_large();
+}
+
+void ZRememberedSet::swap_remset_bitmaps() {
+  assert(previous()->is_empty(), "Previous remset bits should be empty when swapping");
+  current()->iterate([&](BitMap::idx_t index) {
+    previous()->set_bit(index);
+    return true;
+  });
+  current()->clear_large();
+}
+
+ZBitMap::ReverseIterator ZRememberedSet::iterator_reverse_previous() {
+  return ZBitMap::ReverseIterator(previous());
+}
+
+BitMap::Iterator ZRememberedSet::iterator_limited_current(uintptr_t offset, size_t size) {
+  const size_t index = to_index(offset);;
+  const size_t bit_size = to_bit_size(size);
+
+  return BitMap::Iterator(*current(), index, index + bit_size);
+}
+
+ZBitMap::Iterator ZRememberedSet::iterator_limited_previous(uintptr_t offset, size_t size) {
+  const size_t index = to_index(offset);;
+  const size_t bit_size = to_bit_size(size);
+
+  return BitMap::Iterator(*previous(), index, index + bit_size);
+}
+
+size_t ZRememberedSetContainingIterator::to_index(zaddress_unsafe addr) {
+  const uintptr_t local_offset = _page->local_offset(addr);
+  return ZRememberedSet::to_index(local_offset);
+}
+
+zaddress_unsafe ZRememberedSetContainingIterator::to_addr(BitMap::idx_t index) {
+  const uintptr_t local_offset = ZRememberedSet::to_offset(index);
+  return ZOffset::address_unsafe(_page->global_offset(local_offset));
+}
+
+ZRememberedSetContainingIterator::ZRememberedSetContainingIterator(ZPage* page) :
+    _page(page),
+    _remset_iter(page->remset_reverse_iterator_previous()),
+    _obj(zaddress_unsafe::null),
+    _obj_remset_iter(page->remset_reverse_iterator_previous()) {}
+
+bool ZRememberedSetContainingIterator::next(ZRememberedSetContaining* containing) {
+  // Note: to skip having to read the contents of the heap, when collecting the
+  // containing information, this code doesn't read the size of the objects and
+  // therefore doesn't filter out remset bits that belong to dead objects.
+  // The (addr, addr_field) pair will contain the nearest live object, of a
+  // given remset bit. Users of 'containing' need to do the filtering.
+
+  BitMap::idx_t index;
+
+  if (!is_null(_obj)) {
+    // We've already found a remset bit and likely owning object in the main
+    // iterator. Now use that information to skip having to search for the
+    // same object multiple times.
+
+    if (_obj_remset_iter.next(&index)) {
+      containing->_field_addr = to_addr(index);
+      containing->_addr = _obj;
+
+      log_develop_trace(gc, remset)("Remset Containing Obj  index: " PTR_FORMAT " base: " PTR_FORMAT " field: " PTR_FORMAT, index, untype(containing->_addr), untype(containing->_field_addr));
+
+      return true;
+    } else {
+      // No more remset bits in the scanned object
+      _obj = zaddress_unsafe::null;
+    }
+  }
+
+  // At this point, we don't know where the nearest earlier object starts.
+  // Search for the next earlier remset bit, and then search for the likely
+  // owning object.
+  if (_remset_iter.next(&index)) {
+    containing->_field_addr = to_addr(index);
+    containing->_addr = _page->find_base((volatile zpointer*)untype(containing->_field_addr));
+
+    if (is_null(containing->_addr)) {
+      // Found no live object
+      return false;
+    }
+
+    // Found live object. Not necessarily the one that originally owned the remset bit.
+    const BitMap::idx_t obj_index = to_index(containing->_addr);
+
+    log_develop_trace(gc, remset)("Remset Containing Main index: " PTR_FORMAT " base: " PTR_FORMAT " field: " PTR_FORMAT, index, untype(containing->_addr), untype(containing->_field_addr));
+
+    // Don't scan inside the object in the main iterator
+    _remset_iter.reset(obj_index);
+
+    // Scan inside the object iterator
+    _obj = containing->_addr;
+    _obj_remset_iter.reset(obj_index, index);
+
+    return true;
+  }
+
+  return false;
+}
+
+ZRememberedSetContainingInLiveIterator::ZRememberedSetContainingInLiveIterator(ZPage* page) :
+    _iter(page),
+    _addr(zaddress::null),
+    _addr_size(0),
+    _count(0),
+    _count_skipped(0),
+    _page(page) {}
+
+bool ZRememberedSetContainingInLiveIterator::next(ZRememberedSetContaining* containing) {
+  ZRememberedSetContaining local;
+  while (_iter.next(&local)) {
+    const zaddress local_addr = safe(local._addr);
+    if (local_addr != _addr) {
+      _addr = local_addr;
+      _addr_size = ZUtils::object_size(_addr);
+    }
+
+    const size_t field_offset = safe(local._field_addr) - _addr;
+    if (field_offset < _addr_size) {
+      *containing = local;
+      _count++;
+      return true;
+    }
+
+    // Skip field outside object
+    _count_skipped++;
+  }
+
+  // No more entries found
+  return false;
+}
+
+void ZRememberedSetContainingInLiveIterator::print_statistics() const {
+  _page->log_msg(" (remembered iter count: " SIZE_FORMAT " skipped: " SIZE_FORMAT ")", _count, _count_skipped);
+}
diff --git a/src/hotspot/share/gc/z/zRememberedSet.hpp b/src/hotspot/share/gc/z/zRememberedSet.hpp
new file mode 100644
index 00000000000..d18c357f0dd
--- /dev/null
+++ b/src/hotspot/share/gc/z/zRememberedSet.hpp
@@ -0,0 +1,146 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZREMEMBEREDSET_HPP
+#define SHARE_GC_Z_ZREMEMBEREDSET_HPP
+
+#include "gc/z/zAddress.hpp"
+#include "gc/z/zBitMap.hpp"
+
+class OopClosure;
+class ZPage;
+
+struct ZRememberedSetContaining {
+  zaddress_unsafe _field_addr;
+  zaddress_unsafe _addr;
+};
+
+// Iterates over all (object, oop fields) pairs where the field address has
+// been marked as remembered, and fill in that information in a
+// ZRememberedSetContaining
+//
+// Note that it's not guaranteed that _field_addr belongs to the recorded
+// _addr. The entry could denote a stale remembered set field and _addr could
+// just be the nearest object. The users are responsible for filtering that
+// out.
+class ZRememberedSetContainingIterator {
+private:
+  ZPage* const             _page;
+  ZBitMap::ReverseIterator _remset_iter;
+
+  zaddress_unsafe          _obj;
+  ZBitMap::ReverseIterator _obj_remset_iter;
+
+  size_t to_index(zaddress_unsafe addr);
+  zaddress_unsafe to_addr(BitMap::idx_t index);
+
+public:
+  ZRememberedSetContainingIterator(ZPage* page);
+
+  bool next(ZRememberedSetContaining* containing);
+};
+
+// Like ZRememberedSetContainingIterator, but with stale remembered set fields
+// filtered out.
+class ZRememberedSetContainingInLiveIterator {
+private:
+  ZRememberedSetContainingIterator _iter;
+  zaddress                         _addr;
+  size_t                           _addr_size;
+  size_t                           _count;
+  size_t                           _count_skipped;
+  ZPage* const                     _page;
+
+public:
+  ZRememberedSetContainingInLiveIterator(ZPage* page);
+
+  bool next(ZRememberedSetContaining* containing);
+
+  void print_statistics() const;
+};
+
+// The remembered set of a ZPage.
+//
+// There's one bit per potential object field address within the ZPage.
+//
+// New entries are added to the "current" active bitmap, while the
+// "previous" bitmap is used by the GC to find pointers from old
+// gen to young gen.
+class ZRememberedSet {
+  friend class ZRememberedSetContainingIterator;
+
+public:
+  static int _current;
+
+  ZMovableBitMap _bitmap[2];
+
+  CHeapBitMap* current();
+  const CHeapBitMap* current() const;
+
+  CHeapBitMap* previous();
+  const CHeapBitMap* previous() const;
+
+  template <typename Function>
+  void iterate_bitmap(Function function, CHeapBitMap* bitmap);
+
+  static uintptr_t to_offset(BitMap::idx_t index);
+  static BitMap::idx_t to_index(uintptr_t offset);
+  static BitMap::idx_t to_bit_size(size_t size);
+
+public:
+  static void flip();
+
+  ZRememberedSet();
+
+  bool is_initialized() const;
+  void initialize(size_t page_size);
+
+  void resize(size_t page_size);
+
+  bool at_current(uintptr_t offset) const;
+  bool at_previous(uintptr_t offset) const;
+  bool set_current(uintptr_t offset);
+  void unset_non_par_current(uintptr_t offset);
+  void unset_range_non_par_current(uintptr_t offset, size_t size);
+
+  // Visit all set offsets.
+  template <typename Function /* void(uintptr_t offset) */>
+  void iterate_previous(Function function);
+
+  template <typename Function /* void(uintptr_t offset) */>
+  void iterate_current(Function function);
+
+  bool is_cleared_current() const;
+  bool is_cleared_previous() const;
+
+  void clear_all();
+  void clear_current();
+  void clear_previous();
+  void swap_remset_bitmaps();
+
+  ZBitMap::ReverseIterator iterator_reverse_previous();
+  BitMap::Iterator iterator_limited_current(uintptr_t offset, size_t size);
+  BitMap::Iterator iterator_limited_previous(uintptr_t offset, size_t size);
+};
+
+#endif // SHARE_GC_Z_ZREMEMBEREDSET_HPP
diff --git a/src/hotspot/share/gc/z/zRememberedSet.inline.hpp b/src/hotspot/share/gc/z/zRememberedSet.inline.hpp
new file mode 100644
index 00000000000..a4f5ac72075
--- /dev/null
+++ b/src/hotspot/share/gc/z/zRememberedSet.inline.hpp
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZREMEMBEREDSET_INLINE_HPP
+#define SHARE_GC_Z_ZREMEMBEREDSET_INLINE_HPP
+
+#include "gc/z/zRememberedSet.hpp"
+
+#include "utilities/bitMap.inline.hpp"
+
+inline CHeapBitMap* ZRememberedSet::current() {
+  return &_bitmap[_current];
+}
+
+inline const CHeapBitMap* ZRememberedSet::current() const {
+  return &_bitmap[_current];
+}
+
+inline CHeapBitMap* ZRememberedSet::previous() {
+  return &_bitmap[_current ^ 1];
+}
+
+inline const CHeapBitMap* ZRememberedSet::previous() const {
+  return &_bitmap[_current ^ 1];
+}
+
+inline uintptr_t ZRememberedSet::to_offset(BitMap::idx_t index) {
+  // One bit per possible oop* address
+  return index * oopSize;
+}
+
+inline BitMap::idx_t ZRememberedSet::to_index(uintptr_t offset) {
+  // One bit per possible oop* address
+  return offset / oopSize;
+}
+
+inline BitMap::idx_t ZRememberedSet::to_bit_size(size_t size) {
+  return size / oopSize;
+}
+
+inline bool ZRememberedSet::at_current(uintptr_t offset) const {
+  const BitMap::idx_t index = to_index(offset);
+  return current()->at(index);
+}
+
+inline bool ZRememberedSet::at_previous(uintptr_t offset) const {
+  const BitMap::idx_t index = to_index(offset);
+  return previous()->at(index);
+}
+
+inline bool ZRememberedSet::set_current(uintptr_t offset) {
+  const BitMap::idx_t index = to_index(offset);
+  return current()->par_set_bit(index, memory_order_relaxed);
+}
+
+inline void ZRememberedSet::unset_non_par_current(uintptr_t offset) {
+  const BitMap::idx_t index = to_index(offset);
+  current()->clear_bit(index);
+}
+
+inline void ZRememberedSet::unset_range_non_par_current(uintptr_t offset, size_t size) {
+  const BitMap::idx_t start_index = to_index(offset);
+  const BitMap::idx_t end_index = to_index(offset + size);
+  current()->clear_range(start_index, end_index);
+}
+
+template <typename Function>
+void ZRememberedSet::iterate_bitmap(Function function, CHeapBitMap* bitmap) {
+  bitmap->iterate([&](BitMap::idx_t index) {
+    const uintptr_t offset = to_offset(index);
+
+    function(offset);
+
+    return true;
+  });
+}
+
+template <typename Function>
+void ZRememberedSet::iterate_previous(Function function) {
+  iterate_bitmap(function, previous());
+}
+
+template <typename Function>
+void ZRememberedSet::iterate_current(Function function) {
+  iterate_bitmap(function, current());
+}
+
+#endif // SHARE_GC_Z_ZREMEMBEREDSET_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zRootsIterator.cpp b/src/hotspot/share/gc/z/zRootsIterator.cpp
index 08f10fdf0ef..ffa4b468056 100644
--- a/src/hotspot/share/gc/z/zRootsIterator.cpp
+++ b/src/hotspot/share/gc/z/zRootsIterator.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -35,11 +35,59 @@
 #include "runtime/safepoint.hpp"
 #include "utilities/debug.hpp"
 
-static const ZStatSubPhase ZSubPhaseConcurrentRootsOopStorageSet("Concurrent Roots OopStorageSet");
-static const ZStatSubPhase ZSubPhaseConcurrentRootsClassLoaderDataGraph("Concurrent Roots ClassLoaderDataGraph");
-static const ZStatSubPhase ZSubPhaseConcurrentRootsJavaThreads("Concurrent Roots JavaThreads");
-static const ZStatSubPhase ZSubPhaseConcurrentRootsCodeCache("Concurrent Roots CodeCache");
-static const ZStatSubPhase ZSubPhaseConcurrentWeakRootsOopStorageSet("Concurrent Weak Roots OopStorageSet");
+class ZRootStatSubPhase {
+private:
+  ZStatSubPhase _young;
+  ZStatSubPhase _old;
+
+public:
+  ZRootStatSubPhase(const char* name) :
+     _young(name, ZGenerationId::young),
+     _old(name, ZGenerationId::old) {}
+
+  const ZStatSubPhase& young() const { return _young; }
+  const ZStatSubPhase& old() const { return _old; }
+};
+
+static const ZRootStatSubPhase ZSubPhaseConcurrentRootsOopStorageSet("Concurrent Roots OopStorageSet");
+static const ZRootStatSubPhase ZSubPhaseConcurrentRootsClassLoaderDataGraph("Concurrent Roots ClassLoaderDataGraph");
+static const ZRootStatSubPhase ZSubPhaseConcurrentRootsJavaThreads("Concurrent Roots JavaThreads");
+static const ZRootStatSubPhase ZSubPhaseConcurrentRootsCodeCache("Concurrent Roots CodeCache");
+static const ZRootStatSubPhase ZSubPhaseConcurrentWeakRootsOopStorageSet("Concurrent Weak Roots OopStorageSet");
+
+class ZRootStatTimer {
+private:
+  const ZStatPhase*  _phase;
+  const Ticks        _start;
+
+  ZRootStatTimer(const ZStatPhase* phase) :
+      _phase(phase),
+      _start(Ticks::now()) {
+    if (phase != nullptr) {
+      _phase->register_start(nullptr /* timer */, _start);
+    }
+  }
+
+public:
+  ~ZRootStatTimer() {
+    if (_phase != nullptr) {
+      const Ticks end = Ticks::now();
+      _phase->register_end(nullptr /* timer */, _start, end);
+    }
+  }
+
+  static const ZStatSubPhase* calculate_subphase(const ZGenerationIdOptional generation, const ZRootStatSubPhase& subphase) {
+    switch (generation) {
+      case ZGenerationIdOptional::young: return &subphase.young();
+      case ZGenerationIdOptional::old: return &subphase.old();
+      default: return nullptr;
+    }
+  }
+
+public:
+  ZRootStatTimer(const ZRootStatSubPhase& subphase, const ZGenerationIdOptional generation) :
+      ZRootStatTimer(calculate_subphase(generation, subphase)) {}
+};
 
 template <typename Iterator>
 template <typename ClosureType>
@@ -52,91 +100,107 @@ void ZParallelApply<Iterator>::apply(ClosureType* cl) {
   }
 }
 
-ZStrongOopStorageSetIterator::ZStrongOopStorageSetIterator() :
-    _iter() {}
-
-void ZStrongOopStorageSetIterator::apply(OopClosure* cl) {
-  ZStatTimer timer(ZSubPhaseConcurrentRootsOopStorageSet);
+void ZOopStorageSetIteratorStrong::apply(OopClosure* cl) {
+  ZRootStatTimer timer(ZSubPhaseConcurrentWeakRootsOopStorageSet, _generation);
   _iter.oops_do(cl);
 }
 
-void ZStrongCLDsIterator::apply(CLDClosure* cl) {
-  ZStatTimer timer(ZSubPhaseConcurrentRootsClassLoaderDataGraph);
+void ZCLDsIteratorStrong::apply(CLDClosure* cl) {
+  ZRootStatTimer timer(ZSubPhaseConcurrentRootsClassLoaderDataGraph, _generation);
   ClassLoaderDataGraph::always_strong_cld_do(cl);
 }
 
-ZJavaThreadsIterator::ZJavaThreadsIterator() :
-    _threads(),
-    _claimed(0) {}
+void ZCLDsIteratorWeak::apply(CLDClosure* cl) {
+  ZRootStatTimer timer(ZSubPhaseConcurrentRootsClassLoaderDataGraph, _generation);
+  ClassLoaderDataGraph::roots_cld_do(nullptr /* strong */, cl /* weak */);
+}
+
+void ZCLDsIteratorAll::apply(CLDClosure* cl) {
+  ZRootStatTimer timer(ZSubPhaseConcurrentRootsClassLoaderDataGraph, _generation);
+  ClassLoaderDataGraph::cld_do(cl);
+}
 
 uint ZJavaThreadsIterator::claim() {
   return Atomic::fetch_and_add(&_claimed, 1u);
 }
 
 void ZJavaThreadsIterator::apply(ThreadClosure* cl) {
-  ZStatTimer timer(ZSubPhaseConcurrentRootsJavaThreads);
+  ZRootStatTimer timer(ZSubPhaseConcurrentRootsJavaThreads, _generation);
 
   // The resource mark is needed because interpreter oop maps are
   // not reused in concurrent mode. Instead, they are temporary and
   // resource allocated.
-  ResourceMark                 _rm;
+  ResourceMark rm;
 
   for (uint i = claim(); i < _threads.length(); i = claim()) {
     cl->do_thread(_threads.thread_at(i));
   }
 }
 
-ZNMethodsIterator::ZNMethodsIterator() {
-  if (!ClassUnloading) {
-    ZNMethod::nmethods_do_begin();
+ZNMethodsIteratorImpl::ZNMethodsIteratorImpl(ZGenerationIdOptional generation, bool enabled, bool secondary) :
+    _enabled(enabled),
+    _secondary(secondary),
+    _generation(generation) {
+  if (_enabled) {
+    ZNMethod::nmethods_do_begin(secondary);
   }
 }
 
-ZNMethodsIterator::~ZNMethodsIterator() {
-  if (!ClassUnloading) {
-    ZNMethod::nmethods_do_end();
+ZNMethodsIteratorImpl::~ZNMethodsIteratorImpl() {
+  if (_enabled) {
+    ZNMethod::nmethods_do_end(_secondary);
   }
 }
 
-void ZNMethodsIterator::apply(NMethodClosure* cl) {
-  ZStatTimer timer(ZSubPhaseConcurrentRootsCodeCache);
-  ZNMethod::nmethods_do(cl);
+void ZNMethodsIteratorImpl::apply(NMethodClosure* cl) {
+  ZRootStatTimer timer(ZSubPhaseConcurrentRootsCodeCache, _generation);
+  ZNMethod::nmethods_do(_secondary, cl);
 }
 
-ZRootsIterator::ZRootsIterator(int cld_claim) {
-  if (cld_claim != ClassLoaderData::_claim_none) {
-    ClassLoaderDataGraph::verify_claimed_marks_cleared(cld_claim);
-  }
+void ZRootsIteratorStrongColored::apply(OopClosure* cl,
+                                  CLDClosure* cld_cl) {
+  _oop_storage_set_strong.apply(cl);
+  _clds_strong.apply(cld_cl);
 }
 
-void ZRootsIterator::apply(OopClosure* cl,
-                           CLDClosure* cld_cl,
-                           ThreadClosure* thread_cl,
-                           NMethodClosure* nm_cl) {
-  _oop_storage_set.apply(cl);
-  _class_loader_data_graph.apply(cld_cl);
+void ZRootsIteratorStrongUncolored::apply(ThreadClosure* thread_cl,
+                                          NMethodClosure* nm_cl) {
   _java_threads.apply(thread_cl);
   if (!ClassUnloading) {
-    _nmethods.apply(nm_cl);
+    _nmethods_strong.apply(nm_cl);
   }
 }
 
-ZWeakOopStorageSetIterator::ZWeakOopStorageSetIterator() :
-    _iter() {}
+void ZRootsIteratorWeakUncolored::apply(NMethodClosure* nm_cl) {
+  _nmethods_weak.apply(nm_cl);
+}
 
-void ZWeakOopStorageSetIterator::apply(OopClosure* cl) {
-  ZStatTimer timer(ZSubPhaseConcurrentWeakRootsOopStorageSet);
+void ZOopStorageSetIteratorWeak::apply(OopClosure* cl) {
+  ZRootStatTimer timer(ZSubPhaseConcurrentWeakRootsOopStorageSet, _generation);
   _iter.oops_do(cl);
 }
 
-void ZWeakOopStorageSetIterator::report_num_dead() {
+void ZOopStorageSetIteratorWeak::report_num_dead() {
   _iter.report_num_dead();
 }
 
-void ZWeakRootsIterator::report_num_dead() {
-  _oop_storage_set.iter().report_num_dead();
+void ZRootsIteratorWeakColored::report_num_dead() {
+  _oop_storage_set_weak.iter().report_num_dead();
 }
 
-void ZWeakRootsIterator::apply(OopClosure* cl) {
-  _oop_storage_set.apply(cl);
+void ZRootsIteratorWeakColored::apply(OopClosure* cl) {
+  _oop_storage_set_weak.apply(cl);
+}
+
+void ZRootsIteratorAllColored::apply(OopClosure* cl,
+                                     CLDClosure* cld_cl) {
+  _oop_storage_set_strong.apply(cl);
+  _oop_storage_set_weak.apply(cl);
+  _clds_all.apply(cld_cl);
+}
+
+void ZRootsIteratorAllUncolored::apply(ThreadClosure* thread_cl,
+                                       NMethodClosure* nm_cl) {
+  _java_threads.apply(thread_cl);
+  _nmethods_all.apply(nm_cl);
 }
diff --git a/src/hotspot/share/gc/z/zRootsIterator.hpp b/src/hotspot/share/gc/z/zRootsIterator.hpp
index 100089056ad..ca30cf8080c 100644
--- a/src/hotspot/share/gc/z/zRootsIterator.hpp
+++ b/src/hotspot/share/gc/z/zRootsIterator.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,6 +25,7 @@
 #define SHARE_GC_Z_ZROOTSITERATOR_HPP
 
 #include "gc/shared/oopStorageSetParState.hpp"
+#include "gc/z/zGenerationId.hpp"
 #include "logging/log.hpp"
 #include "memory/iterator.hpp"
 #include "runtime/threadSMR.hpp"
@@ -36,8 +37,8 @@ private:
   volatile bool _completed;
 
 public:
-  ZParallelApply() :
-      _iter(),
+  ZParallelApply(ZGenerationIdOptional generation) :
+      _iter(generation),
       _completed(false) {}
 
   template <typename ClosureType>
@@ -48,77 +49,196 @@ public:
   }
 };
 
-class ZStrongOopStorageSetIterator {
+class ZOopStorageSetIteratorStrong {
+private:
   OopStorageSetStrongParState<true /* concurrent */, false /* is_const */> _iter;
+  const ZGenerationIdOptional _generation;
 
 public:
-  ZStrongOopStorageSetIterator();
+  ZOopStorageSetIteratorStrong(ZGenerationIdOptional generation) :
+      _iter(),
+      _generation(generation) {}
 
   void apply(OopClosure* cl);
 };
 
-class ZStrongCLDsIterator {
+class ZOopStorageSetIteratorWeak {
+private:
+  OopStorageSetWeakParState<true /* concurrent */, false /* is_const */> _iter;
+  const ZGenerationIdOptional _generation;
+
 public:
+  ZOopStorageSetIteratorWeak(ZGenerationIdOptional generation) :
+      _iter(),
+      _generation(generation) {}
+
+  void apply(OopClosure* cl);
+
+  void report_num_dead();
+};
+
+class ZCLDsIteratorStrong {
+private:
+  const ZGenerationIdOptional _generation;
+
+public:
+  ZCLDsIteratorStrong(ZGenerationIdOptional generation) :
+      _generation(generation) {}
+
+  void apply(CLDClosure* cl);
+};
+
+class ZCLDsIteratorWeak {
+private:
+  const ZGenerationIdOptional _generation;
+
+public:
+  ZCLDsIteratorWeak(ZGenerationIdOptional generation) :
+      _generation(generation) {}
+
+  void apply(CLDClosure* cl);
+};
+
+class ZCLDsIteratorAll {
+private:
+  const ZGenerationIdOptional _generation;
+
+public:
+  ZCLDsIteratorAll(ZGenerationIdOptional generation) :
+      _generation(generation) {}
+
   void apply(CLDClosure* cl);
 };
 
 class ZJavaThreadsIterator {
 private:
-  ThreadsListHandle _threads;
-  volatile uint     _claimed;
+  ThreadsListHandle           _threads;
+  volatile uint               _claimed;
+  const ZGenerationIdOptional _generation;
 
   uint claim();
 
 public:
-  ZJavaThreadsIterator();
+  ZJavaThreadsIterator(ZGenerationIdOptional generation) :
+      _threads(),
+      _claimed(0),
+      _generation(generation) {}
 
   void apply(ThreadClosure* cl);
 };
 
-class ZNMethodsIterator {
-public:
-  ZNMethodsIterator();
-  ~ZNMethodsIterator();
+class ZNMethodsIteratorImpl {
+private:
+  const bool                  _enabled;
+  const bool                  _secondary;
+  const ZGenerationIdOptional _generation;
 
+protected:
+  ZNMethodsIteratorImpl(ZGenerationIdOptional generation, bool enabled, bool secondary);
+  ~ZNMethodsIteratorImpl();
+
+public:
   void apply(NMethodClosure* cl);
 };
 
-class ZRootsIterator {
+class ZNMethodsIteratorStrong : public ZNMethodsIteratorImpl {
+public:
+  ZNMethodsIteratorStrong(ZGenerationIdOptional generation) :
+      ZNMethodsIteratorImpl(generation, !ClassUnloading /* enabled */, false /* secondary */) {}
+};
+
+class ZNMethodsIteratorWeak : public ZNMethodsIteratorImpl {
+public:
+  ZNMethodsIteratorWeak(ZGenerationIdOptional generation) :
+      ZNMethodsIteratorImpl(generation, true /* enabled */, true /* secondary */) {}
+};
+
+class ZNMethodsIteratorAll : public ZNMethodsIteratorImpl {
+public:
+  ZNMethodsIteratorAll(ZGenerationIdOptional generation) :
+      ZNMethodsIteratorImpl(generation, true /* enabled */, true /* secondary */) {}
+};
+
+class ZRootsIteratorStrongUncolored {
 private:
-  ZParallelApply<ZStrongOopStorageSetIterator> _oop_storage_set;
-  ZParallelApply<ZStrongCLDsIterator>          _class_loader_data_graph;
-  ZParallelApply<ZJavaThreadsIterator>         _java_threads;
-  ZParallelApply<ZNMethodsIterator>            _nmethods;
+  ZParallelApply<ZJavaThreadsIterator>    _java_threads;
+  ZParallelApply<ZNMethodsIteratorStrong> _nmethods_strong;
 
 public:
-  ZRootsIterator(int cld_claim);
+  ZRootsIteratorStrongUncolored(ZGenerationIdOptional generation) :
+      _java_threads(generation),
+      _nmethods_strong(generation) {}
 
-  void apply(OopClosure* cl,
-             CLDClosure* cld_cl,
-             ThreadClosure* thread_cl,
+  void apply(ThreadClosure* thread_cl,
              NMethodClosure* nm_cl);
 };
 
-class ZWeakOopStorageSetIterator {
+class ZRootsIteratorWeakUncolored {
 private:
-  OopStorageSetWeakParState<true /* concurrent */, false /* is_const */> _iter;
+  ZParallelApply<ZNMethodsIteratorWeak> _nmethods_weak;
 
 public:
-  ZWeakOopStorageSetIterator();
+  ZRootsIteratorWeakUncolored(ZGenerationIdOptional generation) :
+      _nmethods_weak(generation) {}
+
+  void apply(NMethodClosure* nm_cl);
+};
+
+class ZRootsIteratorAllUncolored {
+private:
+  ZParallelApply<ZJavaThreadsIterator> _java_threads;
+  ZParallelApply<ZNMethodsIteratorAll> _nmethods_all;
+
+public:
+  ZRootsIteratorAllUncolored(ZGenerationIdOptional generation) :
+      _java_threads(generation),
+      _nmethods_all(generation) {}
+
+  void apply(ThreadClosure* thread_cl,
+             NMethodClosure* nm_cl);
+};
+
+class ZRootsIteratorStrongColored {
+private:
+  ZParallelApply<ZOopStorageSetIteratorStrong> _oop_storage_set_strong;
+  ZParallelApply<ZCLDsIteratorStrong>          _clds_strong;
+
+public:
+  ZRootsIteratorStrongColored(ZGenerationIdOptional generation) :
+      _oop_storage_set_strong(generation),
+      _clds_strong(generation) {}
+
+  void apply(OopClosure* cl,
+             CLDClosure* cld_cl);
+};
+
+class ZRootsIteratorWeakColored {
+private:
+  ZParallelApply<ZOopStorageSetIteratorWeak> _oop_storage_set_weak;
+
+public:
+  ZRootsIteratorWeakColored(ZGenerationIdOptional generation) :
+      _oop_storage_set_weak(generation) {}
 
   void apply(OopClosure* cl);
 
   void report_num_dead();
 };
 
-class ZWeakRootsIterator {
+class ZRootsIteratorAllColored {
 private:
-  ZParallelApply<ZWeakOopStorageSetIterator> _oop_storage_set;
+  ZParallelApply<ZOopStorageSetIteratorStrong> _oop_storage_set_strong;
+  ZParallelApply<ZOopStorageSetIteratorWeak>   _oop_storage_set_weak;
+  ZParallelApply<ZCLDsIteratorAll>             _clds_all;
 
 public:
-  void apply(OopClosure* cl);
+  ZRootsIteratorAllColored(ZGenerationIdOptional generation) :
+      _oop_storage_set_strong(generation),
+      _oop_storage_set_weak(generation),
+      _clds_all(generation) {}
 
-  void report_num_dead();
+  void apply(OopClosure* cl,
+             CLDClosure* cld_cl);
 };
 
 #endif // SHARE_GC_Z_ZROOTSITERATOR_HPP
diff --git a/src/hotspot/share/gc/z/zRuntimeWorkers.cpp b/src/hotspot/share/gc/z/zRuntimeWorkers.cpp
index 568b9b778ce..d11adb5a044 100644
--- a/src/hotspot/share/gc/z/zRuntimeWorkers.cpp
+++ b/src/hotspot/share/gc/z/zRuntimeWorkers.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,39 +24,9 @@
 #include "precompiled.hpp"
 #include "gc/shared/gcLogPrecious.hpp"
 #include "gc/shared/gc_globals.hpp"
-#include "gc/z/zLock.inline.hpp"
 #include "gc/z/zRuntimeWorkers.hpp"
-#include "gc/z/zTask.hpp"
-#include "gc/z/zThread.hpp"
 #include "runtime/java.hpp"
 
-class ZRuntimeWorkersInitializeTask : public WorkerTask {
-private:
-  const uint     _nworkers;
-  uint           _started;
-  ZConditionLock _lock;
-
-public:
-  ZRuntimeWorkersInitializeTask(uint nworkers) :
-      WorkerTask("ZRuntimeWorkersInitializeTask"),
-      _nworkers(nworkers),
-      _started(0),
-      _lock() {}
-
-  virtual void work(uint worker_id) {
-    // Wait for all threads to start
-    ZLocker<ZConditionLock> locker(&_lock);
-    if (++_started == _nworkers) {
-      // All threads started
-      _lock.notify_all();
-    } else {
-      while (_started != _nworkers) {
-        _lock.wait();
-      }
-    }
-  }
-};
-
 ZRuntimeWorkers::ZRuntimeWorkers() :
     _workers("RuntimeWorker",
              ParallelGCThreads) {
@@ -69,11 +39,6 @@ ZRuntimeWorkers::ZRuntimeWorkers() :
   if (_workers.active_workers() != _workers.max_workers()) {
     vm_exit_during_initialization("Failed to create ZRuntimeWorkers");
   }
-
-  // Execute task to reduce latency in early safepoints,
-  // which otherwise would have to take on any warmup costs.
-  ZRuntimeWorkersInitializeTask task(_workers.max_workers());
-  _workers.run_task(&task);
 }
 
 WorkerThreads* ZRuntimeWorkers::workers() {
diff --git a/src/hotspot/share/gc/z/zSafeDelete.hpp b/src/hotspot/share/gc/z/zSafeDelete.hpp
index 62c3495c57b..b0e5f99e57b 100644
--- a/src/hotspot/share/gc/z/zSafeDelete.hpp
+++ b/src/hotspot/share/gc/z/zSafeDelete.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,44 +25,25 @@
 #define SHARE_GC_Z_ZSAFEDELETE_HPP
 
 #include "gc/z/zArray.hpp"
-#include "gc/z/zLock.hpp"
 
 #include <type_traits>
 
 template <typename T>
-class ZSafeDeleteImpl {
+class ZSafeDelete {
 private:
   using ItemT = std::remove_extent_t<T>;
 
-  ZLock*         _lock;
-  uint64_t       _enabled;
-  ZArray<ItemT*> _deferred;
+  ZActivatedArray<T> _deferred;
 
-  bool deferred_delete(ItemT* item);
-  void immediate_delete(ItemT* item);
+  static void immediate_delete(ItemT* item);
 
 public:
-  ZSafeDeleteImpl(ZLock* lock);
+  explicit ZSafeDelete(bool locked = true);
 
   void enable_deferred_delete();
   void disable_deferred_delete();
 
-  void operator()(ItemT* item);
-};
-
-template <typename T>
-class ZSafeDelete : public ZSafeDeleteImpl<T> {
-private:
-  ZLock _lock;
-
-public:
-  ZSafeDelete();
-};
-
-template <typename T>
-class ZSafeDeleteNoLock : public ZSafeDeleteImpl<T> {
-public:
-  ZSafeDeleteNoLock();
+  void schedule_delete(ItemT* item);
 };
 
 #endif // SHARE_GC_Z_ZSAFEDELETE_HPP
diff --git a/src/hotspot/share/gc/z/zSafeDelete.inline.hpp b/src/hotspot/share/gc/z/zSafeDelete.inline.hpp
index 460193827e0..e894d555c0a 100644
--- a/src/hotspot/share/gc/z/zSafeDelete.inline.hpp
+++ b/src/hotspot/share/gc/z/zSafeDelete.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,29 +27,15 @@
 #include "gc/z/zSafeDelete.hpp"
 
 #include "gc/z/zArray.inline.hpp"
-#include "utilities/debug.hpp"
 
 #include <type_traits>
 
 template <typename T>
-ZSafeDeleteImpl<T>::ZSafeDeleteImpl(ZLock* lock) :
-    _lock(lock),
-    _enabled(0),
-    _deferred() {}
+ZSafeDelete<T>::ZSafeDelete(bool locked) :
+    _deferred(locked) {}
 
 template <typename T>
-bool ZSafeDeleteImpl<T>::deferred_delete(ItemT* item) {
-  ZLocker<ZLock> locker(_lock);
-  if (_enabled > 0) {
-    _deferred.append(item);
-    return true;
-  }
-
-  return false;
-}
-
-template <typename T>
-void ZSafeDeleteImpl<T>::immediate_delete(ItemT* item) {
+void ZSafeDelete<T>::immediate_delete(ItemT* item) {
   if (std::is_array<T>::value) {
     delete [] item;
   } else {
@@ -58,43 +44,20 @@ void ZSafeDeleteImpl<T>::immediate_delete(ItemT* item) {
 }
 
 template <typename T>
-void ZSafeDeleteImpl<T>::enable_deferred_delete() {
-  ZLocker<ZLock> locker(_lock);
-  _enabled++;
+void ZSafeDelete<T>::enable_deferred_delete() {
+  _deferred.activate();
 }
 
 template <typename T>
-void ZSafeDeleteImpl<T>::disable_deferred_delete() {
-  ZArray<ItemT*> deferred;
+void ZSafeDelete<T>::disable_deferred_delete() {
+  _deferred.deactivate_and_apply(immediate_delete);
+}
 
-  {
-    ZLocker<ZLock> locker(_lock);
-    assert(_enabled > 0, "Invalid state");
-    if (--_enabled == 0) {
-      deferred.swap(&_deferred);
-    }
-  }
-
-  ZArrayIterator<ItemT*> iter(&deferred);
-  for (ItemT* item; iter.next(&item);) {
+template <typename T>
+void ZSafeDelete<T>::schedule_delete(ItemT* item) {
+  if (!_deferred.add_if_activated(item)) {
     immediate_delete(item);
   }
 }
 
-template <typename T>
-void ZSafeDeleteImpl<T>::operator()(ItemT* item) {
-  if (!deferred_delete(item)) {
-    immediate_delete(item);
-  }
-}
-
-template <typename T>
-ZSafeDelete<T>::ZSafeDelete() :
-    ZSafeDeleteImpl<T>(&_lock),
-    _lock() {}
-
-template <typename T>
-ZSafeDeleteNoLock<T>::ZSafeDeleteNoLock() :
-    ZSafeDeleteImpl<T>(NULL) {}
-
 #endif // SHARE_GC_Z_ZSAFEDELETE_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zServiceability.cpp b/src/hotspot/share/gc/z/zServiceability.cpp
index c708d0d0c47..b20da7ff5bf 100644
--- a/src/hotspot/share/gc/z/zServiceability.cpp
+++ b/src/hotspot/share/gc/z/zServiceability.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,11 +25,32 @@
 #include "gc/shared/generationCounters.hpp"
 #include "gc/shared/hSpaceCounters.hpp"
 #include "gc/z/zCollectedHeap.hpp"
+#include "gc/z/zDriver.hpp"
 #include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zServiceability.hpp"
 #include "memory/metaspaceCounters.hpp"
 #include "runtime/perfData.hpp"
 
+struct ZMemoryUsageInfo {
+  size_t _young_used;
+  size_t _young_capacity;
+  size_t _old_used;
+  size_t _old_capacity;
+};
+
+static ZMemoryUsageInfo compute_memory_usage_info() {
+  const size_t capacity = ZHeap::heap()->capacity();
+  const size_t old_used = ZHeap::heap()->used_old();
+  const size_t young_used = ZHeap::heap()->used_young();
+
+  ZMemoryUsageInfo info;
+  info._old_used = MIN2(old_used, capacity);
+  info._old_capacity = info._old_used;
+  info._young_capacity = capacity - info._old_capacity;
+  info._young_used = MIN2(young_used, info._young_capacity);
+  return info;
+}
+
 class ZGenerationCounters : public GenerationCounters {
 public:
   ZGenerationCounters(const char* name, int ordinal, int spaces,
@@ -45,122 +66,189 @@ public:
 // Class to expose perf counters used by jstat.
 class ZServiceabilityCounters : public CHeapObj<mtGC> {
 private:
-  ZGenerationCounters _generation_counters;
-  HSpaceCounters      _space_counters;
-  CollectorCounters   _collector_counters;
+  ZGenerationCounters _generation_young_counters;
+  ZGenerationCounters _generation_old_counters;
+  HSpaceCounters      _space_young_counters;
+  HSpaceCounters      _space_old_counters;
+  CollectorCounters   _minor_collection_counters;
+  CollectorCounters   _major_collection_counters;
 
 public:
-  ZServiceabilityCounters(size_t min_capacity, size_t max_capacity);
+  ZServiceabilityCounters(size_t initial_capacity, size_t min_capacity, size_t max_capacity);
 
-  CollectorCounters* collector_counters();
+  CollectorCounters* collector_counters(bool minor);
 
   void update_sizes();
 };
 
-ZServiceabilityCounters::ZServiceabilityCounters(size_t min_capacity, size_t max_capacity) :
+ZServiceabilityCounters::ZServiceabilityCounters(size_t initial_capacity, size_t min_capacity, size_t max_capacity) :
+    // generation.0
+    _generation_young_counters(
+        "young"          /* name */,
+        0                /* ordinal */,
+        1                /* spaces */,
+        min_capacity     /* min_capacity */,
+        max_capacity     /* max_capacity */,
+        initial_capacity /* curr_capacity */),
     // generation.1
-    _generation_counters("old"        /* name */,
-                         1            /* ordinal */,
-                         1            /* spaces */,
-                         min_capacity /* min_capacity */,
-                         max_capacity /* max_capacity */,
-                         min_capacity /* curr_capacity */),
+    _generation_old_counters(
+        "old"        /* name */,
+        1            /* ordinal */,
+        1            /* spaces */,
+        0            /* min_capacity */,
+        max_capacity /* max_capacity */,
+        0            /* curr_capacity */),
+    // generation.0.space.0
+    _space_young_counters(
+        _generation_young_counters.name_space(),
+        "space"          /* name */,
+        0                /* ordinal */,
+        max_capacity     /* max_capacity */,
+        initial_capacity /* init_capacity */),
     // generation.1.space.0
-    _space_counters(_generation_counters.name_space(),
-                    "space"      /* name */,
-                    0            /* ordinal */,
-                    max_capacity /* max_capacity */,
-                    min_capacity /* init_capacity */),
+    _space_old_counters(
+        _generation_old_counters.name_space(),
+        "space"      /* name */,
+        0            /* ordinal */,
+        max_capacity /* max_capacity */,
+        0            /* init_capacity */),
+    // gc.collector.0
+    _minor_collection_counters(
+        "ZGC minor collection pauses" /* name */,
+        0                             /* ordinal */),
     // gc.collector.2
-    _collector_counters("Z concurrent cycle pauses" /* name */,
-                        2                           /* ordinal */) {}
+    _major_collection_counters(
+        "ZGC major collection pauses" /* name */,
+        2                             /* ordinal */) {}
 
-CollectorCounters* ZServiceabilityCounters::collector_counters() {
-  return &_collector_counters;
+CollectorCounters* ZServiceabilityCounters::collector_counters(bool minor) {
+  return minor
+      ? &_minor_collection_counters
+      : &_major_collection_counters;
 }
 
 void ZServiceabilityCounters::update_sizes() {
   if (UsePerfData) {
-    const size_t capacity = ZHeap::heap()->capacity();
-    const size_t used = MIN2(ZHeap::heap()->used(), capacity);
-
-    _generation_counters.update_capacity(capacity);
-    _space_counters.update_capacity(capacity);
-    _space_counters.update_used(used);
+    const ZMemoryUsageInfo info = compute_memory_usage_info();
+    _generation_young_counters.update_capacity(info._young_capacity);
+    _generation_old_counters.update_capacity(info._old_capacity);
+    _space_young_counters.update_capacity(info._young_capacity);
+    _space_young_counters.update_used(info._young_used);
+    _space_old_counters.update_capacity(info._old_capacity);
+    _space_old_counters.update_used(info._old_used);
 
     MetaspaceCounters::update_performance_counters();
   }
 }
 
-ZServiceabilityMemoryPool::ZServiceabilityMemoryPool(size_t min_capacity, size_t max_capacity) :
-    CollectedMemoryPool("ZHeap",
+ZServiceabilityMemoryPool::ZServiceabilityMemoryPool(const char* name, ZGenerationId id, size_t min_capacity, size_t max_capacity) :
+    CollectedMemoryPool(name,
                         min_capacity,
                         max_capacity,
-                        true /* support_usage_threshold */) {}
+                        id == ZGenerationId::old /* support_usage_threshold */),
+    _generation_id(id) {}
 
 size_t ZServiceabilityMemoryPool::used_in_bytes() {
-  return ZHeap::heap()->used();
+  return ZHeap::heap()->used_generation(_generation_id);
 }
 
 MemoryUsage ZServiceabilityMemoryPool::get_memory_usage() {
-  const size_t committed = ZHeap::heap()->capacity();
-  const size_t used      = MIN2(ZHeap::heap()->used(), committed);
+  const ZMemoryUsageInfo info = compute_memory_usage_info();
 
-  return MemoryUsage(initial_size(), used, committed, max_size());
+  if (_generation_id == ZGenerationId::young) {
+    return MemoryUsage(initial_size(), info._young_used, info._young_capacity, max_size());
+  } else {
+    return MemoryUsage(initial_size(), info._old_used, info._old_capacity, max_size());
+  }
 }
 
 ZServiceabilityMemoryManager::ZServiceabilityMemoryManager(const char* name,
-                                                           ZServiceabilityMemoryPool* pool) :
+                                                           MemoryPool* young_memory_pool,
+                                                           MemoryPool* old_memory_pool) :
     GCMemoryManager(name) {
-  add_pool(pool);
+  add_pool(young_memory_pool);
+  add_pool(old_memory_pool);
 }
 
-ZServiceability::ZServiceability(size_t min_capacity, size_t max_capacity) :
+ZServiceability::ZServiceability(size_t initial_capacity,
+                                 size_t min_capacity,
+                                 size_t max_capacity) :
+    _initial_capacity(initial_capacity),
     _min_capacity(min_capacity),
     _max_capacity(max_capacity),
-    _memory_pool(_min_capacity, _max_capacity),
-    _cycle_memory_manager("ZGC Cycles", &_memory_pool),
-    _pause_memory_manager("ZGC Pauses", &_memory_pool),
-    _counters(NULL) {}
+    _young_memory_pool("ZGC Young Generation", ZGenerationId::young, _min_capacity, _max_capacity),
+    _old_memory_pool("ZGC Old Generation", ZGenerationId::old, 0, _max_capacity),
+    _minor_cycle_memory_manager("ZGC Minor Cycles", &_young_memory_pool, &_old_memory_pool),
+    _major_cycle_memory_manager("ZGC Major Cycles", &_young_memory_pool, &_old_memory_pool),
+    _minor_pause_memory_manager("ZGC Minor Pauses", &_young_memory_pool, &_old_memory_pool),
+    _major_pause_memory_manager("ZGC Major Pauses", &_young_memory_pool, &_old_memory_pool),
+    _counters(nullptr) {
+}
 
 void ZServiceability::initialize() {
-  _counters = new ZServiceabilityCounters(_min_capacity, _max_capacity);
+  _counters = new ZServiceabilityCounters(_initial_capacity, _min_capacity, _max_capacity);
 }
 
-MemoryPool* ZServiceability::memory_pool() {
-  return &_memory_pool;
+MemoryPool* ZServiceability::memory_pool(ZGenerationId id) {
+  return id == ZGenerationId::young
+      ? &_young_memory_pool
+      : &_old_memory_pool;
 }
 
-GCMemoryManager* ZServiceability::cycle_memory_manager() {
-  return &_cycle_memory_manager;
+GCMemoryManager* ZServiceability::cycle_memory_manager(bool minor) {
+  return minor
+      ? &_minor_cycle_memory_manager
+      : &_major_cycle_memory_manager;
 }
 
-GCMemoryManager* ZServiceability::pause_memory_manager() {
-  return &_pause_memory_manager;
+GCMemoryManager* ZServiceability::pause_memory_manager(bool minor) {
+  return minor
+      ? &_minor_pause_memory_manager
+      : &_major_pause_memory_manager;
 }
 
 ZServiceabilityCounters* ZServiceability::counters() {
   return _counters;
 }
 
-ZServiceabilityCycleTracer::ZServiceabilityCycleTracer() :
-    _memory_manager_stats(ZHeap::heap()->serviceability_cycle_memory_manager(),
-                          ZCollectedHeap::heap()->gc_cause(),
+bool ZServiceabilityCycleTracer::_minor_is_active;
+
+ZServiceabilityCycleTracer::ZServiceabilityCycleTracer(bool minor) :
+    _memory_manager_stats(ZHeap::heap()->serviceability_cycle_memory_manager(minor),
+                          minor ? ZDriver::minor()->gc_cause() : ZDriver::major()->gc_cause(),
                           "end of GC cycle",
-                          true  /* allMemoryPoolsAffected */,
-                          true  /* recordGCBeginTime */,
-                          true  /* recordPreGCUsage */,
-                          true  /* recordPeakUsage */,
-                          true  /* recordPostGCUsage */,
-                          true  /* recordAccumulatedGCTime */,
-                          true  /* recordGCEndTime */,
-                          true  /* countCollection */) {}
+                          true /* allMemoryPoolsAffected */,
+                          true /* recordGCBeginTime */,
+                          true /* recordPreGCUsage */,
+                          true /* recordPeakUsage */,
+                          true /* recordPostGCUsage */,
+                          true /* recordAccumulatedGCTime */,
+                          true /* recordGCEndTime */,
+                          true /* countCollection */) {
+  _minor_is_active = minor;
+}
+
+ZServiceabilityCycleTracer::~ZServiceabilityCycleTracer() {
+  _minor_is_active = false;
+}
+
+bool ZServiceabilityCycleTracer::minor_is_active() {
+  return _minor_is_active;
+}
+
+bool ZServiceabilityPauseTracer::minor_is_active() const {
+  // We report pauses at the minor/major collection level instead
+  // of the young/old level. At the call-site where ZServiceabilityPauseTracer
+  // is used, we don't have that information readily available, so
+  // we let ZServiceabilityCycleTracer keep track of that.
+  return ZServiceabilityCycleTracer::minor_is_active();
+}
 
 ZServiceabilityPauseTracer::ZServiceabilityPauseTracer() :
     _svc_gc_marker(SvcGCMarker::CONCURRENT),
-    _counters_stats(ZHeap::heap()->serviceability_counters()->collector_counters()),
-    _memory_manager_stats(ZHeap::heap()->serviceability_pause_memory_manager(),
-                          ZCollectedHeap::heap()->gc_cause(),
+    _counters_stats(ZHeap::heap()->serviceability_counters()->collector_counters(minor_is_active())),
+    _memory_manager_stats(ZHeap::heap()->serviceability_pause_memory_manager(minor_is_active()),
+                          minor_is_active() ? ZDriver::minor()->gc_cause() : ZDriver::major()->gc_cause(),
                           "end of GC pause",
                           true  /* allMemoryPoolsAffected */,
                           true  /* recordGCBeginTime */,
diff --git a/src/hotspot/share/gc/z/zServiceability.hpp b/src/hotspot/share/gc/z/zServiceability.hpp
index 5d03184d244..2a3b56248df 100644
--- a/src/hotspot/share/gc/z/zServiceability.hpp
+++ b/src/hotspot/share/gc/z/zServiceability.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,6 +26,7 @@
 
 #include "gc/shared/collectorCounters.hpp"
 #include "gc/shared/gcVMOperations.hpp"
+#include "gc/z/zGenerationId.hpp"
 #include "memory/allocation.hpp"
 #include "services/memoryManager.hpp"
 #include "services/memoryPool.hpp"
@@ -34,8 +35,11 @@
 class ZServiceabilityCounters;
 
 class ZServiceabilityMemoryPool : public CollectedMemoryPool {
+private:
+  const ZGenerationId _generation_id;
+
 public:
-  ZServiceabilityMemoryPool(size_t min_capacity, size_t max_capacity);
+  ZServiceabilityMemoryPool(const char* name, ZGenerationId id, size_t min_capacity, size_t max_capacity);
 
   virtual size_t used_in_bytes();
   virtual MemoryUsage get_memory_usage();
@@ -44,35 +48,47 @@ public:
 class ZServiceabilityMemoryManager : public GCMemoryManager {
 public:
   ZServiceabilityMemoryManager(const char* name,
-                               ZServiceabilityMemoryPool* pool);
+                               MemoryPool* young_memory_pool,
+                               MemoryPool* old_memory_pool);
 };
 
 class ZServiceability {
 private:
+  const size_t                 _initial_capacity;
   const size_t                 _min_capacity;
   const size_t                 _max_capacity;
-  ZServiceabilityMemoryPool    _memory_pool;
-  ZServiceabilityMemoryManager _cycle_memory_manager;
-  ZServiceabilityMemoryManager _pause_memory_manager;
+  ZServiceabilityMemoryPool    _young_memory_pool;
+  ZServiceabilityMemoryPool    _old_memory_pool;
+  ZServiceabilityMemoryManager _minor_cycle_memory_manager;
+  ZServiceabilityMemoryManager _major_cycle_memory_manager;
+  ZServiceabilityMemoryManager _minor_pause_memory_manager;
+  ZServiceabilityMemoryManager _major_pause_memory_manager;
   ZServiceabilityCounters*     _counters;
 
 public:
-  ZServiceability(size_t min_capacity, size_t max_capacity);
+  ZServiceability(size_t initial_capacity,
+                  size_t min_capacity,
+                  size_t max_capacity);
 
   void initialize();
 
-  MemoryPool* memory_pool();
-  GCMemoryManager* cycle_memory_manager();
-  GCMemoryManager* pause_memory_manager();
+  MemoryPool* memory_pool(ZGenerationId id);
+  GCMemoryManager* cycle_memory_manager(bool minor);
+  GCMemoryManager* pause_memory_manager(bool minor);
   ZServiceabilityCounters* counters();
 };
 
 class ZServiceabilityCycleTracer : public StackObj {
 private:
+  static bool _minor_is_active;
+
   TraceMemoryManagerStats _memory_manager_stats;
 
 public:
-  ZServiceabilityCycleTracer();
+  ZServiceabilityCycleTracer(bool minor);
+  ~ZServiceabilityCycleTracer();
+
+  static bool minor_is_active();
 };
 
 class ZServiceabilityPauseTracer : public StackObj {
@@ -81,6 +97,8 @@ private:
   TraceCollectorStats     _counters_stats;
   TraceMemoryManagerStats _memory_manager_stats;
 
+  bool minor_is_active() const;
+
 public:
   ZServiceabilityPauseTracer();
   ~ZServiceabilityPauseTracer();
diff --git a/src/hotspot/share/gc/z/zStackChunkGCData.hpp b/src/hotspot/share/gc/z/zStackChunkGCData.hpp
new file mode 100644
index 00000000000..fefdc559193
--- /dev/null
+++ b/src/hotspot/share/gc/z/zStackChunkGCData.hpp
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZSTACKCHUNKGCDATA_HPP
+#define SHARE_GC_Z_ZSTACKCHUNKGCDATA_HPP
+
+#include "oops/oopsHierarchy.hpp"
+
+class ZStackChunkGCData {
+private:
+  // The implicit color of all oops when the chunk was recently allocated
+  uintptr_t _color;
+
+  static ZStackChunkGCData* data(stackChunkOop chunk);
+
+public:
+  static void initialize(stackChunkOop chunk);
+  static uintptr_t color(stackChunkOop chunk);
+};
+
+#endif // SHARE_GC_Z_ZSTACKCHUNKGCDATA_HPP
diff --git a/src/hotspot/share/gc/z/zStackChunkGCData.inline.hpp b/src/hotspot/share/gc/z/zStackChunkGCData.inline.hpp
new file mode 100644
index 00000000000..0a5ca86fa68
--- /dev/null
+++ b/src/hotspot/share/gc/z/zStackChunkGCData.inline.hpp
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZSTACKCHUNKGCDATA_INLINE_HPP
+#define SHARE_GC_Z_ZSTACKCHUNKGCDATA_INLINE_HPP
+
+#include "gc/z/zStackChunkGCData.hpp"
+
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "oops/stackChunkOop.inline.hpp"
+
+inline ZStackChunkGCData* ZStackChunkGCData::data(stackChunkOop chunk) {
+  return reinterpret_cast<ZStackChunkGCData*>(chunk->gc_data());
+}
+
+inline void ZStackChunkGCData::initialize(stackChunkOop chunk) {
+  data(chunk)->_color = ZPointerStoreGoodMask;
+}
+
+inline uintptr_t ZStackChunkGCData::color(stackChunkOop chunk) {
+  return data(chunk)->_color;
+}
+
+#endif // SHARE_GC_Z_ZSTACKCHUNKGCDATA_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zStackWatermark.cpp b/src/hotspot/share/gc/z/zStackWatermark.cpp
index 7280f350e80..ed0c054858d 100644
--- a/src/hotspot/share/gc/z/zStackWatermark.cpp
+++ b/src/hotspot/share/gc/z/zStackWatermark.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,13 +24,17 @@
 #include "precompiled.hpp"
 #include "gc/z/zAddress.hpp"
 #include "gc/z/zBarrier.inline.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zStackWatermark.hpp"
-#include "gc/z/zThread.inline.hpp"
+#include "gc/z/zStoreBarrierBuffer.hpp"
 #include "gc/z/zThreadLocalAllocBuffer.hpp"
 #include "gc/z/zThreadLocalData.hpp"
+#include "gc/z/zUncoloredRoot.inline.hpp"
 #include "gc/z/zVerify.hpp"
 #include "memory/resourceArea.inline.hpp"
 #include "runtime/frame.inline.hpp"
+#include "runtime/stackWatermark.hpp"
+#include "runtime/thread.hpp"
 #include "utilities/preserveException.hpp"
 
 ZOnStackCodeBlobClosure::ZOnStackCodeBlobClosure() :
@@ -38,7 +42,7 @@ ZOnStackCodeBlobClosure::ZOnStackCodeBlobClosure() :
 
 void ZOnStackCodeBlobClosure::do_code_blob(CodeBlob* cb) {
   nmethod* const nm = cb->as_nmethod_or_null();
-  if (nm != NULL) {
+  if (nm != nullptr) {
     const bool result = _bs_nm->nmethod_entry_barrier(nm);
     assert(result, "NMethod on-stack must be alive");
   }
@@ -49,51 +53,163 @@ ThreadLocalAllocStats& ZStackWatermark::stats() {
 }
 
 uint32_t ZStackWatermark::epoch_id() const {
-  return *ZAddressBadMaskHighOrderBitsAddr;
+  return *ZPointerStoreGoodMaskLowOrderBitsAddr;
 }
 
 ZStackWatermark::ZStackWatermark(JavaThread* jt) :
-    StackWatermark(jt, StackWatermarkKind::gc, *ZAddressBadMaskHighOrderBitsAddr),
-    _jt_cl(),
-    _cb_cl(),
+    StackWatermark(jt, StackWatermarkKind::gc, *ZPointerStoreGoodMaskLowOrderBitsAddr),
+    // First watermark is fake and setup to be replaced at next phase shift
+    _old_watermarks{{ZPointerStoreBadMask, 1}, {}, {}},
+    _old_watermarks_newest(0),
     _stats() {}
 
-OopClosure* ZStackWatermark::closure_from_context(void* context) {
-  if (context != NULL) {
-    assert(ZThread::is_worker(), "Unexpected thread passing in context: " PTR_FORMAT, p2i(context));
-    return reinterpret_cast<OopClosure*>(context);
+bool ZColorWatermark::covers(const ZColorWatermark& other) const {
+  if (_watermark == 0) {
+    // This watermark was completed
+    return true;
+  }
+
+  if (other._watermark == 0) {
+    // The other watermark was completed
+    return false;
+  }
+
+  // Compare the two
+  return _watermark >= other._watermark;
+}
+
+uintptr_t ZStackWatermark::prev_head_color() const {
+  return _old_watermarks[_old_watermarks_newest]._color;
+}
+
+uintptr_t ZStackWatermark::prev_frame_color(const frame& fr) const {
+  for (int i = _old_watermarks_newest; i >= 0; i--) {
+    const ZColorWatermark ow = _old_watermarks[i];
+    if (ow._watermark == 0 || uintptr_t(fr.sp()) <= ow._watermark) {
+      return ow._color;
+    }
+  }
+
+  fatal("Found no matching previous color for the frame");
+  return 0;
+}
+
+void ZStackWatermark::save_old_watermark() {
+  assert(StackWatermarkState::epoch(_state) != ZStackWatermark::epoch_id(), "Shouldn't be here otherwise");
+
+  // Previous color
+  const uintptr_t prev_color = StackWatermarkState::epoch(_state);
+
+  // If the prev_color is still the last saved color watermark, then processing has not started.
+  const bool prev_processing_started = prev_color != prev_head_color();
+
+  if (!prev_processing_started) {
+    // Nothing was processed in the previous phase, so there's no need to save a watermark for it.
+    // Must have been a remapped phase, the other phases are explicitly completed by the GC.
+    assert((prev_color & ZPointerRemapped) != 0, "Unexpected color: " PTR_FORMAT, prev_color);
+    return;
+  }
+
+  // Previous watermark
+  const uintptr_t prev_watermark = StackWatermarkState::is_done(_state) ? 0 : last_processed_raw();
+
+  // Create a new color watermark to describe the old watermark
+  const ZColorWatermark cw = { prev_color, prev_watermark };
+
+  // Find the location of the oldest watermark that it covers, and thus can replace
+  int replace = -1;
+  for (int i = 0; i <= _old_watermarks_newest; i++) {
+    if (cw.covers(_old_watermarks[i])) {
+      replace = i;
+      break;
+    }
+  }
+
+  // Update top
+  if (replace != -1) {
+    // Found one to replace
+    _old_watermarks_newest = replace;
   } else {
-    return &_jt_cl;
+    // Found none too replace - push it to the top
+    _old_watermarks_newest++;
+    assert(_old_watermarks_newest < _old_watermarks_max, "Unexpected amount of old watermarks");
+  }
+
+  // Install old watermark
+  _old_watermarks[_old_watermarks_newest] = cw;
+}
+
+class ZStackWatermarkProcessOopClosure : public ZUncoloredRootClosure {
+private:
+  const ZUncoloredRoot::RootFunction _function;
+  const uintptr_t                    _color;
+
+  static ZUncoloredRoot::RootFunction select_function(void* context) {
+    if (context == nullptr) {
+      return ZUncoloredRoot::process;
+    }
+
+    assert(Thread::current()->is_Worker_thread(), "Unexpected thread passing in context: " PTR_FORMAT, p2i(context));
+    return reinterpret_cast<ZUncoloredRoot::RootFunction>(context);
+  }
+
+public:
+  ZStackWatermarkProcessOopClosure(void* context, uintptr_t color) :
+      _function(select_function(context)), _color(color) {}
+
+  virtual void do_root(zaddress_unsafe* p) {
+    _function(p, _color);
+  }
+};
+
+void ZStackWatermark::process_head(void* context) {
+  const uintptr_t color = prev_head_color();
+
+  ZStackWatermarkProcessOopClosure cl(context, color);
+  ZOnStackCodeBlobClosure cb_cl;
+
+  _jt->oops_do_no_frames(&cl, &cb_cl);
+
+  zaddress_unsafe* const invisible_root = ZThreadLocalData::invisible_root(_jt);
+  if (invisible_root != nullptr) {
+    ZUncoloredRoot::process_invisible(invisible_root, color);
   }
 }
 
 void ZStackWatermark::start_processing_impl(void* context) {
-  // Verify the head (no_frames) of the thread is bad before fixing it.
-  ZVerify::verify_thread_head_bad(_jt);
+  save_old_watermark();
 
   // Process the non-frame part of the thread
-  _jt->oops_do_no_frames(closure_from_context(context), &_cb_cl);
-  ZThreadLocalData::do_invisible_root(_jt, ZBarrier::load_barrier_on_invisible_root_oop_field);
+  process_head(context);
 
   // Verification of frames is done after processing of the "head" (no_frames).
   // The reason is that the exception oop is fiddled with during frame processing.
-  ZVerify::verify_thread_frames_bad(_jt);
+  // ZVerify::verify_thread_frames_bad(_jt);
 
-  // Update thread local address bad mask
-  ZThreadLocalData::set_address_bad_mask(_jt, ZAddressBadMask);
+  // Update thread-local masks
+  ZThreadLocalData::set_load_bad_mask(_jt, ZPointerLoadBadMask);
+  ZThreadLocalData::set_load_good_mask(_jt, ZPointerLoadGoodMask);
+  ZThreadLocalData::set_mark_bad_mask(_jt, ZPointerMarkBadMask);
+  ZThreadLocalData::set_store_bad_mask(_jt, ZPointerStoreBadMask);
+  ZThreadLocalData::set_store_good_mask(_jt, ZPointerStoreGoodMask);
+  ZThreadLocalData::set_nmethod_disarmed(_jt, ZPointerStoreGoodMask);
 
   // Retire TLAB
-  if (ZGlobalPhase == ZPhaseMark) {
+  if (ZGeneration::young()->is_phase_mark() || ZGeneration::old()->is_phase_mark()) {
     ZThreadLocalAllocBuffer::retire(_jt, &_stats);
-  } else {
-    ZThreadLocalAllocBuffer::remap(_jt);
   }
 
+  // Prepare store barrier buffer for new GC phase
+  ZThreadLocalData::store_barrier_buffer(_jt)->on_new_phase();
+
   // Publishes the processing start to concurrent threads
   StackWatermark::start_processing_impl(context);
 }
 
 void ZStackWatermark::process(const frame& fr, RegisterMap& register_map, void* context) {
-  ZVerify::verify_frame_bad(fr, register_map);
-  fr.oops_do(closure_from_context(context), &_cb_cl, &register_map, DerivedPointerIterationMode::_directly);
+  const uintptr_t color = prev_frame_color(fr);
+  ZStackWatermarkProcessOopClosure cl(context, color);
+  ZOnStackCodeBlobClosure cb_cl;
+
+  fr.oops_do(&cl, &cb_cl, &register_map, DerivedPointerIterationMode::_directly);
 }
diff --git a/src/hotspot/share/gc/z/zStackWatermark.hpp b/src/hotspot/share/gc/z/zStackWatermark.hpp
index e40b24f4623..042ba4dd3f8 100644
--- a/src/hotspot/share/gc/z/zStackWatermark.hpp
+++ b/src/hotspot/share/gc/z/zStackWatermark.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,6 +28,7 @@
 #include "gc/shared/barrierSetNMethod.hpp"
 #include "gc/shared/threadLocalAllocBuffer.hpp"
 #include "gc/z/zBarrier.hpp"
+#include "gc/z/zUncoloredRoot.hpp"
 #include "memory/allocation.hpp"
 #include "memory/iterator.hpp"
 #include "oops/oopsHierarchy.hpp"
@@ -47,13 +48,28 @@ public:
   ZOnStackCodeBlobClosure();
 };
 
+struct ZColorWatermark {
+  uintptr_t _color;
+  uintptr_t _watermark;
+
+  bool covers(const ZColorWatermark& other) const;
+};
+
 class ZStackWatermark : public StackWatermark {
 private:
-  ZLoadBarrierOopClosure  _jt_cl;
-  ZOnStackCodeBlobClosure _cb_cl;
-  ThreadLocalAllocStats   _stats;
+  // Stores old watermarks, which describes the
+  // colors of the non-processed part of the stack.
+  const static int      _old_watermarks_max = 3;
+  ZColorWatermark       _old_watermarks[_old_watermarks_max];
+  int                   _old_watermarks_newest;
 
-  OopClosure* closure_from_context(void* context);
+  ThreadLocalAllocStats _stats;
+
+  uintptr_t prev_head_color() const;
+  uintptr_t prev_frame_color(const frame& fr) const;
+  void save_old_watermark();
+
+  void process_head(void* context);
 
   virtual uint32_t epoch_id() const;
   virtual void start_processing_impl(void* context);
diff --git a/src/hotspot/share/gc/z/zStat.cpp b/src/hotspot/share/gc/z/zStat.cpp
index 540c171ae9f..5c2a87938af 100644
--- a/src/hotspot/share/gc/z/zStat.cpp
+++ b/src/hotspot/share/gc/z/zStat.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,19 +25,22 @@
 #include "gc/shared/gc_globals.hpp"
 #include "gc/z/zAbort.inline.hpp"
 #include "gc/z/zCollectedHeap.hpp"
+#include "gc/z/zDirector.hpp"
+#include "gc/z/zDriver.hpp"
 #include "gc/z/zCPU.inline.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zNMethodTable.hpp"
 #include "gc/z/zPageAllocator.inline.hpp"
 #include "gc/z/zRelocationSetSelector.inline.hpp"
 #include "gc/z/zStat.hpp"
-#include "gc/z/zThread.inline.hpp"
 #include "gc/z/zTracer.inline.hpp"
 #include "gc/z/zUtils.hpp"
 #include "memory/metaspaceUtils.hpp"
 #include "memory/resourceArea.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/os.hpp"
+#include "runtime/thread.hpp"
 #include "runtime/timer.hpp"
 #include "utilities/align.hpp"
 #include "utilities/debug.hpp"
@@ -240,7 +243,7 @@ public:
 // Stat unit printers
 //
 void ZStatUnitTime(LogTargetHandle log, const ZStatSampler& sampler, const ZStatSamplerHistory& history) {
-  log.print(" %10s: %-41s "
+  log.print(" %16s: %-41s "
             "%9.3f / %-9.3f "
             "%9.3f / %-9.3f "
             "%9.3f / %-9.3f "
@@ -258,7 +261,7 @@ void ZStatUnitTime(LogTargetHandle log, const ZStatSampler& sampler, const ZStat
 }
 
 void ZStatUnitBytes(LogTargetHandle log, const ZStatSampler& sampler, const ZStatSamplerHistory& history) {
-  log.print(" %10s: %-41s "
+  log.print(" %16s: %-41s "
             UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
             UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
             UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
@@ -276,7 +279,7 @@ void ZStatUnitBytes(LogTargetHandle log, const ZStatSampler& sampler, const ZSta
 }
 
 void ZStatUnitThreads(LogTargetHandle log, const ZStatSampler& sampler, const ZStatSamplerHistory& history) {
-  log.print(" %10s: %-41s "
+  log.print(" %16s: %-41s "
             UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
             UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
             UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
@@ -294,7 +297,7 @@ void ZStatUnitThreads(LogTargetHandle log, const ZStatSampler& sampler, const ZS
 }
 
 void ZStatUnitBytesPerSecond(LogTargetHandle log, const ZStatSampler& sampler, const ZStatSamplerHistory& history) {
-  log.print(" %10s: %-41s "
+  log.print(" %16s: %-41s "
             UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
             UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
             UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
@@ -312,7 +315,7 @@ void ZStatUnitBytesPerSecond(LogTargetHandle log, const ZStatSampler& sampler, c
 }
 
 void ZStatUnitOpsPerSecond(LogTargetHandle log, const ZStatSampler& sampler, const ZStatSamplerHistory& history) {
-  log.print(" %10s: %-41s "
+  log.print(" %16s: %-41s "
             UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
             UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
             UINT64_FORMAT_W(9) " / " UINT64_FORMAT_W(-9) " "
@@ -380,7 +383,7 @@ uint32_t ZStatValue::id() const {
 // Stat iterable value
 //
 template <typename T> uint32_t ZStatIterableValue<T>::_count = 0;
-template <typename T> T*       ZStatIterableValue<T>::_first = NULL;
+template <typename T> T*       ZStatIterableValue<T>::_first = nullptr;
 
 template <typename T>
 ZStatIterableValue<T>::ZStatIterableValue(const char* group,
@@ -399,16 +402,16 @@ T* ZStatIterableValue<T>::insert() const {
 template <typename T>
 void ZStatIterableValue<T>::sort() {
   T* first_unsorted = _first;
-  _first = NULL;
+  _first = nullptr;
 
-  while (first_unsorted != NULL) {
+  while (first_unsorted != nullptr) {
     T* const value = first_unsorted;
     first_unsorted = value->_next;
-    value->_next = NULL;
+    value->_next = nullptr;
 
     T** current = &_first;
 
-    while (*current != NULL) {
+    while (*current != nullptr) {
       // First sort by group, then by name
       const int group_cmp = strcmp((*current)->group(), value->group());
       if ((group_cmp > 0) || (group_cmp == 0 && strcmp((*current)->name(), value->name()) > 0)) {
@@ -589,7 +592,6 @@ void ZStatMMU::print() {
 //
 // Stat phases
 //
-ConcurrentGCTimer ZStatPhase::_timer;
 
 ZStatPhase::ZStatPhase(const char* group, const char* name) :
     _sampler(group, name, ZStatUnitTime) {}
@@ -620,79 +622,148 @@ void ZStatPhase::log_end(LogTargetHandle log, const Tickspan& duration, bool thr
   }
 }
 
-ConcurrentGCTimer* ZStatPhase::timer() {
-  return &_timer;
-}
-
 const char* ZStatPhase::name() const {
   return _sampler.name();
 }
 
-ZStatPhaseCycle::ZStatPhaseCycle(const char* name) :
-    ZStatPhase("Collector", name) {}
+ZStatPhaseCollection::ZStatPhaseCollection(const char* name, bool minor) :
+    ZStatPhase(minor ? "Minor Collection" : "Major Collection", name),
+    _minor(minor) {}
 
-void ZStatPhaseCycle::register_start(const Ticks& start) const {
-  timer()->register_gc_start(start);
-
-  ZTracer::tracer()->report_gc_start(ZCollectedHeap::heap()->gc_cause(), start);
-
-  ZCollectedHeap::heap()->print_heap_before_gc();
-  ZCollectedHeap::heap()->trace_heap_before_gc(ZTracer::tracer());
-
-  log_info(gc, start)("Garbage Collection (%s)",
-                       GCCause::to_string(ZCollectedHeap::heap()->gc_cause()));
+GCTracer* ZStatPhaseCollection::jfr_tracer() const {
+  return _minor
+      ? ZDriver::minor()->jfr_tracer()
+      : ZDriver::major()->jfr_tracer();
 }
 
-void ZStatPhaseCycle::register_end(const Ticks& start, const Ticks& end) const {
+void ZStatPhaseCollection::set_used_at_start(size_t used) const {
+  if (_minor) {
+    ZDriver::minor()->set_used_at_start(used);
+  } else {
+    ZDriver::major()->set_used_at_start(used);
+  }
+}
+
+size_t ZStatPhaseCollection::used_at_start() const {
+  return _minor
+      ? ZDriver::minor()->used_at_start()
+      : ZDriver::major()->used_at_start();
+}
+
+void ZStatPhaseCollection::register_start(ConcurrentGCTimer* timer, const Ticks& start) const {
+  const GCCause::Cause cause = _minor ? ZDriver::minor()->gc_cause() : ZDriver::major()->gc_cause();
+
+  timer->register_gc_start(start);
+
+  jfr_tracer()->report_gc_start(cause, start);
+  ZCollectedHeap::heap()->trace_heap_before_gc(jfr_tracer());
+
+  set_used_at_start(ZHeap::heap()->used());
+
+  log_info(gc)("%s (%s)", name(), GCCause::to_string(cause));
+}
+
+void ZStatPhaseCollection::register_end(ConcurrentGCTimer* timer, const Ticks& start, const Ticks& end) const {
+  const GCCause::Cause cause = _minor ? ZDriver::minor()->gc_cause() : ZDriver::major()->gc_cause();
+
   if (ZAbort::should_abort()) {
-    log_info(gc)("Garbage Collection (%s) Aborted",
-                 GCCause::to_string(ZCollectedHeap::heap()->gc_cause()));
+    log_info(gc)("%s (%s) Aborted", name(), GCCause::to_string(cause));
     return;
   }
 
-  timer()->register_gc_end(end);
+  timer->register_gc_end(end);
 
-  ZCollectedHeap::heap()->print_heap_after_gc();
-  ZCollectedHeap::heap()->trace_heap_after_gc(ZTracer::tracer());
-
-  ZTracer::tracer()->report_gc_end(end, timer()->time_partitions());
+  jfr_tracer()->report_gc_end(end, timer->time_partitions());
+  ZCollectedHeap::heap()->trace_heap_after_gc(jfr_tracer());
 
   const Tickspan duration = end - start;
   ZStatSample(_sampler, duration.value());
 
+  const size_t used_at_end = ZHeap::heap()->used();
+
+  log_info(gc)("%s (%s) " ZSIZE_FMT "->" ZSIZE_FMT " %.3fs",
+               name(),
+               GCCause::to_string(cause),
+               ZSIZE_ARGS(used_at_start()),
+               ZSIZE_ARGS(used_at_end),
+               duration.seconds());
+}
+
+ZStatPhaseGeneration::ZStatPhaseGeneration(const char* name, ZGenerationId id) :
+    ZStatPhase(id == ZGenerationId::old ? "Old Generation" : "Young Generation", name),
+    _id(id) {}
+
+ZGenerationTracer* ZStatPhaseGeneration::jfr_tracer() const {
+  return _id == ZGenerationId::young
+      ? ZGeneration::young()->jfr_tracer()
+      : ZGeneration::old()->jfr_tracer();
+}
+
+void ZStatPhaseGeneration::register_start(ConcurrentGCTimer* timer, const Ticks& start) const {
+  ZCollectedHeap::heap()->print_heap_before_gc();
+
+  jfr_tracer()->report_start(start);
+
+  log_info(gc, phases)("%s", name());
+}
+
+void ZStatPhaseGeneration::register_end(ConcurrentGCTimer* timer, const Ticks& start, const Ticks& end) const {
+  if (ZAbort::should_abort()) {
+    log_info(gc, phases)("%s Aborted", name());
+    return;
+  }
+
+  jfr_tracer()->report_end(end);
+
+  ZCollectedHeap::heap()->print_heap_after_gc();
+
+  const Tickspan duration = end - start;
+  ZStatSample(_sampler, duration.value());
+
+  ZGeneration* const generation = ZGeneration::generation(_id);
+
+  generation->stat_heap()->print_stalls();
   ZStatLoad::print();
   ZStatMMU::print();
-  ZStatMark::print();
+  generation->stat_mark()->print();
   ZStatNMethods::print();
   ZStatMetaspace::print();
-  ZStatReferences::print();
-  ZStatRelocation::print();
-  ZStatHeap::print();
+  if (generation->is_old()) {
+    ZStatReferences::print();
+  }
 
-  log_info(gc)("Garbage Collection (%s) " ZSIZE_FMT "->" ZSIZE_FMT,
-               GCCause::to_string(ZCollectedHeap::heap()->gc_cause()),
-               ZSIZE_ARGS(ZStatHeap::used_at_mark_start()),
-               ZSIZE_ARGS(ZStatHeap::used_at_relocate_end()));
+  generation->stat_relocation()->print_page_summary();
+  if (generation->is_young()) {
+    generation->stat_relocation()->print_age_table();
+  }
+
+  generation->stat_heap()->print(generation);
+
+  log_info(gc, phases)("%s " ZSIZE_FMT "->" ZSIZE_FMT " %.3fs",
+                       name(),
+                       ZSIZE_ARGS(generation->stat_heap()->used_at_collection_start()),
+                       ZSIZE_ARGS(generation->stat_heap()->used_at_collection_end()),
+                       duration.seconds());
 }
 
 Tickspan ZStatPhasePause::_max;
 
-ZStatPhasePause::ZStatPhasePause(const char* name) :
-    ZStatPhase("Phase", name) {}
+ZStatPhasePause::ZStatPhasePause(const char* name, ZGenerationId id) :
+    ZStatPhase(id == ZGenerationId::young ? "Young Pause" : "Old Pause", name) {}
 
 const Tickspan& ZStatPhasePause::max() {
   return _max;
 }
 
-void ZStatPhasePause::register_start(const Ticks& start) const {
-  timer()->register_gc_pause_start(name(), start);
+void ZStatPhasePause::register_start(ConcurrentGCTimer* timer, const Ticks& start) const {
+  timer->register_gc_pause_start(name(), start);
 
   LogTarget(Debug, gc, phases, start) log;
   log_start(log);
 }
 
-void ZStatPhasePause::register_end(const Ticks& start, const Ticks& end) const {
-  timer()->register_gc_pause_end(end);
+void ZStatPhasePause::register_end(ConcurrentGCTimer* timer, const Ticks& start, const Ticks& end) const {
+  timer->register_gc_pause_end(end);
 
   const Tickspan duration = end - start;
   ZStatSample(_sampler, duration.value());
@@ -709,22 +780,22 @@ void ZStatPhasePause::register_end(const Ticks& start, const Ticks& end) const {
   log_end(log, duration);
 }
 
-ZStatPhaseConcurrent::ZStatPhaseConcurrent(const char* name) :
-    ZStatPhase("Phase", name) {}
+ZStatPhaseConcurrent::ZStatPhaseConcurrent(const char* name, ZGenerationId id) :
+    ZStatPhase(id == ZGenerationId::young ? "Young Phase" : "Old Phase", name) {}
 
-void ZStatPhaseConcurrent::register_start(const Ticks& start) const {
-  timer()->register_gc_concurrent_start(name(), start);
+void ZStatPhaseConcurrent::register_start(ConcurrentGCTimer* timer, const Ticks& start) const {
+  timer->register_gc_concurrent_start(name(), start);
 
   LogTarget(Debug, gc, phases, start) log;
   log_start(log);
 }
 
-void ZStatPhaseConcurrent::register_end(const Ticks& start, const Ticks& end) const {
+void ZStatPhaseConcurrent::register_end(ConcurrentGCTimer* timer, const Ticks& start, const Ticks& end) const {
   if (ZAbort::should_abort()) {
     return;
   }
 
-  timer()->register_gc_concurrent_end(end);
+  timer->register_gc_concurrent_end(end);
 
   const Tickspan duration = end - start;
   ZStatSample(_sampler, duration.value());
@@ -733,11 +804,16 @@ void ZStatPhaseConcurrent::register_end(const Ticks& start, const Ticks& end) co
   log_end(log, duration);
 }
 
-ZStatSubPhase::ZStatSubPhase(const char* name) :
-    ZStatPhase("Subphase", name) {}
+ZStatSubPhase::ZStatSubPhase(const char* name, ZGenerationId id) :
+    ZStatPhase(id == ZGenerationId::young ? "Young Subphase" : "Old Subphase", name) {}
 
-void ZStatSubPhase::register_start(const Ticks& start) const {
-  if (ZThread::is_worker()) {
+void ZStatSubPhase::register_start(ConcurrentGCTimer* timer, const Ticks& start) const {
+  if (timer != nullptr) {
+    assert(!Thread::current()->is_Worker_thread(), "Unexpected timer value");
+    timer->register_gc_phase_start(name(), start);
+  }
+
+  if (Thread::current()->is_Worker_thread()) {
     LogTarget(Trace, gc, phases, start) log;
     log_start(log, true /* thread */);
   } else {
@@ -746,17 +822,22 @@ void ZStatSubPhase::register_start(const Ticks& start) const {
   }
 }
 
-void ZStatSubPhase::register_end(const Ticks& start, const Ticks& end) const {
+void ZStatSubPhase::register_end(ConcurrentGCTimer* timer, const Ticks& start, const Ticks& end) const {
   if (ZAbort::should_abort()) {
     return;
   }
 
-  ZTracer::tracer()->report_thread_phase(name(), start, end);
+  if (timer != nullptr) {
+    assert(!Thread::current()->is_Worker_thread(), "Unexpected timer value");
+    timer->register_gc_phase_end(end);
+  }
+
+  ZTracer::report_thread_phase(name(), start, end);
 
   const Tickspan duration = end - start;
   ZStatSample(_sampler, duration.value());
 
-  if (ZThread::is_worker()) {
+  if (Thread::current()->is_Worker_thread()) {
     LogTarget(Trace, gc, phases) log;
     log_end(log, duration, true /* thread */);
   } else {
@@ -770,15 +851,15 @@ ZStatCriticalPhase::ZStatCriticalPhase(const char* name, bool verbose) :
     _counter("Critical", name, ZStatUnitOpsPerSecond),
     _verbose(verbose) {}
 
-void ZStatCriticalPhase::register_start(const Ticks& start) const {
+void ZStatCriticalPhase::register_start(ConcurrentGCTimer* timer, const Ticks& start) const {
   // This is called from sensitive contexts, for example before an allocation stall
   // has been resolved. This means we must not access any oops in here since that
   // could lead to infinite recursion. Without access to the thread name we can't
   // really log anything useful here.
 }
 
-void ZStatCriticalPhase::register_end(const Ticks& start, const Ticks& end) const {
-  ZTracer::tracer()->report_thread_phase(name(), start, end);
+void ZStatCriticalPhase::register_end(ConcurrentGCTimer* timer, const Ticks& start, const Ticks& end) const {
+  ZTracer::report_thread_phase(name(), start, end);
 
   const Tickspan duration = end - start;
   ZStatSample(_sampler, duration.value());
@@ -793,10 +874,16 @@ void ZStatCriticalPhase::register_end(const Ticks& start, const Ticks& end) cons
   }
 }
 
-//
-// Stat timer
-//
-THREAD_LOCAL uint32_t ZStatTimerDisable::_active = 0;
+ZStatTimerYoung::ZStatTimerYoung(const ZStatPhase& phase) :
+    ZStatTimer(phase, ZGeneration::young()->gc_timer()) {}
+
+ZStatTimerOld::ZStatTimerOld(const ZStatPhase& phase) :
+    ZStatTimer(phase, ZGeneration::old()->gc_timer()) {}
+
+ZStatTimerWorker::ZStatTimerWorker(const ZStatPhase& phase) :
+    ZStatTimer(phase, nullptr /* gc_timer */) {
+  assert(Thread::current()->is_Worker_thread(), "Should only be called by worker thread");
+}
 
 //
 // Stat sample/inc
@@ -824,14 +911,14 @@ void ZStatSample(const ZStatSampler& sampler, uint64_t value) {
     max = prev_max;
   }
 
-  ZTracer::tracer()->report_stat_sampler(sampler, value);
+  ZTracer::report_stat_sampler(sampler, value);
 }
 
 void ZStatInc(const ZStatCounter& counter, uint64_t increment) {
   ZStatCounterData* const cpu_data = counter.get();
   const uint64_t value = Atomic::add(&cpu_data->_counter, increment);
 
-  ZTracer::tracer()->report_stat_counter(counter, increment, value);
+  ZTracer::report_stat_counter(counter, increment, value);
 }
 
 void ZStatInc(const ZStatUnsampledCounter& counter, uint64_t increment) {
@@ -840,36 +927,88 @@ void ZStatInc(const ZStatUnsampledCounter& counter, uint64_t increment) {
 }
 
 //
-// Stat allocation rate
+// Stat mutator allocation rate
 //
-const ZStatUnsampledCounter ZStatAllocRate::_counter("Allocation Rate");
-TruncatedSeq                ZStatAllocRate::_samples(ZStatAllocRate::sample_hz);
-TruncatedSeq                ZStatAllocRate::_rate(ZStatAllocRate::sample_hz);
+ZLock*          ZStatMutatorAllocRate::_stat_lock;
+jlong           ZStatMutatorAllocRate::_last_sample_time;
+volatile size_t ZStatMutatorAllocRate::_sampling_granule;
+volatile size_t ZStatMutatorAllocRate::_allocated_since_sample;
+TruncatedSeq    ZStatMutatorAllocRate::_samples_time(100);
+TruncatedSeq    ZStatMutatorAllocRate::_samples_bytes(100);
+TruncatedSeq    ZStatMutatorAllocRate::_rate(100);
 
-const ZStatUnsampledCounter& ZStatAllocRate::counter() {
-  return _counter;
+void ZStatMutatorAllocRate::initialize() {
+  _last_sample_time = os::elapsed_counter();
+  _stat_lock = new ZLock();
+  update_sampling_granule();
 }
 
-uint64_t ZStatAllocRate::sample_and_reset() {
-  const ZStatCounterData bytes_per_sample = _counter.collect_and_reset();
-  _samples.add(bytes_per_sample._counter);
+void ZStatMutatorAllocRate::update_sampling_granule() {
+  const size_t sampling_heap_granules = 128;
+  const size_t soft_max_capacity = ZHeap::heap()->soft_max_capacity();
+  _sampling_granule = align_up(soft_max_capacity / sampling_heap_granules, ZGranuleSize);
+}
 
-  const uint64_t bytes_per_second = _samples.sum();
+void ZStatMutatorAllocRate::sample_allocation(size_t allocation_bytes) {
+  const size_t allocated = Atomic::add(&_allocated_since_sample, allocation_bytes);
+
+  if (allocated < Atomic::load(&_sampling_granule)) {
+    // No need for sampling yet
+    return;
+  }
+
+  if (!_stat_lock->try_lock()) {
+    // Someone beat us to it
+    return;
+  }
+
+  const size_t allocated_sample = Atomic::load(&_allocated_since_sample);
+
+  if (allocated_sample < _sampling_granule) {
+    // Someone beat us to it
+    _stat_lock->unlock();
+    return;
+  }
+
+  const jlong now = os::elapsed_counter();
+  const jlong elapsed = now - _last_sample_time;
+
+  if (elapsed <= 0) {
+    // Avoid sampling nonsense allocation rates
+    _stat_lock->unlock();
+    return;
+  }
+
+  Atomic::sub(&_allocated_since_sample, allocated_sample);
+
+  _samples_time.add(elapsed);
+  _samples_bytes.add(allocated_sample);
+
+  const double last_sample_bytes = _samples_bytes.sum();
+  const double elapsed_time = _samples_time.sum();
+
+  const double elapsed_seconds = elapsed_time / os::elapsed_frequency();
+  const double bytes_per_second = double(last_sample_bytes) / elapsed_seconds;
   _rate.add(bytes_per_second);
 
-  return bytes_per_second;
+  update_sampling_granule();
+
+  _last_sample_time = now;
+
+  log_debug(gc, alloc)("Mutator Allocation Rate: %.1fMB/s Predicted: %.1fMB/s, Avg: %.1f(+/-%.1f)MB/s",
+                       bytes_per_second / M,
+                       _rate.predict_next() / M,
+                       _rate.avg() / M,
+                       _rate.sd() / M);
+
+  _stat_lock->unlock();
+
+  ZDirector::evaluate_rules();
 }
 
-double ZStatAllocRate::predict() {
-  return _rate.predict_next();
-}
-
-double ZStatAllocRate::avg() {
-  return _rate.avg();
-}
-
-double ZStatAllocRate::sd() {
-  return _rate.sd();
+ZStatMutatorAllocRateStats ZStatMutatorAllocRate::stats() {
+  ZLocker<ZLock> locker(_stat_lock);
+  return {_rate.avg(), _rate.predict_next(), _rate.sd()};
 }
 
 //
@@ -879,16 +1018,17 @@ ZStat::ZStat() :
     _metronome(sample_hz) {
   set_name("ZStat");
   create_and_start();
+  ZStatMutatorAllocRate::initialize();
 }
 
 void ZStat::sample_and_collect(ZStatSamplerHistory* history) const {
   // Sample counters
-  for (const ZStatCounter* counter = ZStatCounter::first(); counter != NULL; counter = counter->next()) {
+  for (const ZStatCounter* counter = ZStatCounter::first(); counter != nullptr; counter = counter->next()) {
     counter->sample_and_reset();
   }
 
   // Collect samples
-  for (const ZStatSampler* sampler = ZStatSampler::first(); sampler != NULL; sampler = sampler->next()) {
+  for (const ZStatSampler* sampler = ZStatSampler::first(); sampler != nullptr; sampler = sampler->next()) {
     ZStatSamplerHistory& sampler_history = history[sampler->id()];
     sampler_history.add(sampler->collect_and_reset());
   }
@@ -913,7 +1053,7 @@ void ZStat::print(LogTargetHandle log, const ZStatSamplerHistory* history) const
   log.print("                                                             Last 10s              Last 10m              Last 10h                Total");
   log.print("                                                             Avg / Max             Avg / Max             Avg / Max             Avg / Max");
 
-  for (const ZStatSampler* sampler = ZStatSampler::first(); sampler != NULL; sampler = sampler->next()) {
+  for (const ZStatSampler* sampler = ZStatSampler::first(); sampler != nullptr; sampler = sampler->next()) {
     const ZStatSamplerHistory& sampler_history = history[sampler->id()];
     const ZStatUnitPrinter printer = sampler->printer();
     printer(log, *sampler, sampler_history);
@@ -922,9 +1062,9 @@ void ZStat::print(LogTargetHandle log, const ZStatSamplerHistory* history) const
   log.print("=========================================================================================================================================================");
 }
 
-void ZStat::run_service() {
+void ZStat::run_thread() {
   ZStatSamplerHistory* const history = new ZStatSamplerHistory[ZStatSampler::count()];
-  LogTarget(Info, gc, stats) log;
+  LogTarget(Debug, gc, stats) log;
 
   ZStatSampler::sort();
 
@@ -936,10 +1076,16 @@ void ZStat::run_service() {
     }
   }
 
+  // At exit print the final stats
+  LogTarget(Info, gc, stats) exit_log;
+  if (exit_log.is_enabled()) {
+    print(exit_log, history);
+  }
+
   delete [] history;
 }
 
-void ZStat::stop_service() {
+void ZStat::terminate() {
   _metronome.stop();
 }
 
@@ -1072,61 +1218,77 @@ public:
 //
 // Stat cycle
 //
-uint64_t  ZStatCycle::_nwarmup_cycles = 0;
-Ticks     ZStatCycle::_start_of_last;
-Ticks     ZStatCycle::_end_of_last;
-NumberSeq ZStatCycle::_serial_time(0.7 /* alpha */);
-NumberSeq ZStatCycle::_parallelizable_time(0.7 /* alpha */);
-uint      ZStatCycle::_last_active_workers = 0;
+ZStatCycle::ZStatCycle() :
+    _stat_lock(),
+    _nwarmup_cycles(0),
+    _start_of_last(),
+    _end_of_last(),
+    _cycle_intervals(0.7 /* alpha */),
+    _serial_time(0.7 /* alpha */),
+    _parallelizable_time(0.7 /* alpha */),
+    _parallelizable_duration(0.7 /* alpha */),
+    _last_active_workers(0.0) {
+}
 
 void ZStatCycle::at_start() {
+  ZLocker<ZLock> locker(&_stat_lock);
   _start_of_last = Ticks::now();
 }
 
-void ZStatCycle::at_end(GCCause::Cause cause, uint active_workers) {
+void ZStatCycle::at_end(ZStatWorkers* stat_workers, bool record_stats) {
+  ZLocker<ZLock> locker(&_stat_lock);
+  const Ticks end_of_last = _end_of_last;
   _end_of_last = Ticks::now();
 
-  if (cause == GCCause::_z_warmup) {
+  if (ZDriver::major()->gc_cause() == GCCause::_z_warmup && _nwarmup_cycles < 3) {
     _nwarmup_cycles++;
   }
 
-  _last_active_workers = active_workers;
-
   // Calculate serial and parallelizable GC cycle times
   const double duration = (_end_of_last - _start_of_last).seconds();
-  const double workers_duration = ZStatWorkers::get_and_reset_duration();
+  const double workers_duration = stat_workers->get_and_reset_duration();
+  const double workers_time = stat_workers->get_and_reset_time();
   const double serial_time = duration - workers_duration;
-  const double parallelizable_time = workers_duration * active_workers;
-  _serial_time.add(serial_time);
-  _parallelizable_time.add(parallelizable_time);
+
+  _last_active_workers = workers_time / workers_duration;
+
+  if (record_stats) {
+    _serial_time.add(serial_time);
+    _parallelizable_time.add(workers_time);
+    _parallelizable_duration.add(workers_duration);
+    if (end_of_last.value() != 0) {
+      const double cycle_interval = (_end_of_last - end_of_last).seconds();
+      _cycle_intervals.add(cycle_interval);
+    }
+  }
 }
 
 bool ZStatCycle::is_warm() {
   return _nwarmup_cycles >= 3;
 }
 
-uint64_t ZStatCycle::nwarmup_cycles() {
-  return _nwarmup_cycles;
-}
-
 bool ZStatCycle::is_time_trustable() {
   // The times are considered trustable if we
   // have completed at least one warmup cycle.
   return _nwarmup_cycles > 0;
 }
 
-const AbsSeq& ZStatCycle::serial_time() {
-  return _serial_time;
-}
-
-const AbsSeq& ZStatCycle::parallelizable_time() {
-  return _parallelizable_time;
-}
-
-uint ZStatCycle::last_active_workers() {
+double ZStatCycle::last_active_workers() {
   return _last_active_workers;
 }
 
+double ZStatCycle::duration_since_start() {
+  const Ticks start = _start_of_last;
+  if (start.value() == 0) {
+    // No end recorded yet, return time since VM start
+    return 0.0;
+  }
+
+  const Ticks now = Ticks::now();
+  const Tickspan duration_since_start = now - start;
+  return duration_since_start.seconds();
+}
+
 double ZStatCycle::time_since_last() {
   if (_end_of_last.value() == 0) {
     // No end recorded yet, return time since VM start
@@ -1138,63 +1300,145 @@ double ZStatCycle::time_since_last() {
   return time_since_last.seconds();
 }
 
+ZStatCycleStats ZStatCycle::stats() {
+  ZLocker<ZLock> locker(&_stat_lock);
+
+  return {
+    is_warm(),
+    _nwarmup_cycles,
+    is_time_trustable(),
+    time_since_last(),
+    last_active_workers(),
+    duration_since_start(),
+    _cycle_intervals.davg(),
+    _serial_time.davg(),
+    _serial_time.dsd(),
+    _parallelizable_time.davg(),
+    _parallelizable_time.dsd(),
+    _parallelizable_duration.davg(),
+    _parallelizable_duration.dsd()
+  };
+}
+
 //
 // Stat workers
 //
-Ticks ZStatWorkers::_start_of_last;
-Tickspan ZStatWorkers::_accumulated_duration;
+ZStatWorkers::ZStatWorkers() :
+    _stat_lock(),
+    _active_workers(0),
+    _start_of_last(),
+    _accumulated_duration(),
+    _accumulated_time() {}
 
-void ZStatWorkers::at_start() {
+void ZStatWorkers::at_start(uint active_workers) {
+  ZLocker<ZLock> locker(&_stat_lock);
   _start_of_last = Ticks::now();
+  _active_workers = active_workers;
 }
 
 void ZStatWorkers::at_end() {
+  ZLocker<ZLock> locker(&_stat_lock);
   const Ticks now = Ticks::now();
   const Tickspan duration = now - _start_of_last;
+  Tickspan time = duration;
+  for (uint i = 1; i < _active_workers; ++i) {
+    time += duration;
+  }
+  _accumulated_time += time;
   _accumulated_duration += duration;
+  _active_workers = 0;
+}
+
+double ZStatWorkers::accumulated_time() {
+  const uint nworkers = _active_workers;
+  const Ticks now = Ticks::now();
+  const Ticks start = _start_of_last;
+  Tickspan time = _accumulated_time;
+  if (nworkers != 0) {
+    for (uint i = 0; i < nworkers; ++i) {
+      time += now - start;
+    }
+  }
+  return time.seconds();
+}
+
+double ZStatWorkers::accumulated_duration() {
+  const Ticks now = Ticks::now();
+  const Ticks start = _start_of_last;
+  Tickspan duration = _accumulated_duration;
+  if (_active_workers != 0) {
+    duration += now - start;
+  }
+  return duration.seconds();
+}
+
+uint ZStatWorkers::active_workers() {
+  return _active_workers;
 }
 
 double ZStatWorkers::get_and_reset_duration() {
+  ZLocker<ZLock> locker(&_stat_lock);
   const double duration = _accumulated_duration.seconds();
   const Ticks now = Ticks::now();
   _accumulated_duration = now - now;
   return duration;
 }
 
+double ZStatWorkers::get_and_reset_time() {
+  ZLocker<ZLock> locker(&_stat_lock);
+  const double time = _accumulated_time.seconds();
+  const Ticks now = Ticks::now();
+  _accumulated_time = now - now;
+  return time;
+}
+
+ZStatWorkersStats ZStatWorkers::stats() {
+  ZLocker<ZLock> locker(&_stat_lock);
+  return {
+    accumulated_time(),
+    accumulated_duration()
+  };
+}
+
 //
 // Stat load
 //
 void ZStatLoad::print() {
   double loadavg[3] = {};
   os::loadavg(loadavg, ARRAY_SIZE(loadavg));
-  log_info(gc, load)("Load: %.2f/%.2f/%.2f", loadavg[0], loadavg[1], loadavg[2]);
+  log_info(gc, load)("Load: %.2f (%.0f%%) / %.2f (%.0f%%) / %.2f (%.0f%%)",
+                     loadavg[0], percent_of(loadavg[0], (double) ZCPU::count()),
+                     loadavg[1], percent_of(loadavg[1], (double) ZCPU::count()),
+                     loadavg[2], percent_of(loadavg[2], (double) ZCPU::count()));
 }
 
 //
 // Stat mark
 //
-size_t ZStatMark::_nstripes;
-size_t ZStatMark::_nproactiveflush;
-size_t ZStatMark::_nterminateflush;
-size_t ZStatMark::_ntrycomplete;
-size_t ZStatMark::_ncontinue;
-size_t ZStatMark::_mark_stack_usage;
+ZStatMark::ZStatMark() :
+    _nstripes(),
+    _nproactiveflush(),
+    _nterminateflush(),
+    _ntrycomplete(),
+    _ncontinue(),
+    _mark_stack_usage() {
+}
 
-void ZStatMark::set_at_mark_start(size_t nstripes) {
+void ZStatMark::at_mark_start(size_t nstripes) {
   _nstripes = nstripes;
 }
 
-void ZStatMark::set_at_mark_end(size_t nproactiveflush,
-                                size_t nterminateflush,
-                                size_t ntrycomplete,
-                                size_t ncontinue) {
+void ZStatMark::at_mark_end(size_t nproactiveflush,
+                            size_t nterminateflush,
+                            size_t ntrycomplete,
+                            size_t ncontinue) {
   _nproactiveflush = nproactiveflush;
   _nterminateflush = nterminateflush;
   _ntrycomplete = ntrycomplete;
   _ncontinue = ncontinue;
 }
 
-void ZStatMark::set_at_mark_free(size_t mark_stack_usage) {
+void ZStatMark::at_mark_free(size_t mark_stack_usage) {
   _mark_stack_usage = mark_stack_usage;
 }
 
@@ -1217,45 +1461,163 @@ void ZStatMark::print() {
 //
 // Stat relocation
 //
-ZRelocationSetSelectorStats ZStatRelocation::_selector_stats;
-size_t                      ZStatRelocation::_forwarding_usage;
-size_t                      ZStatRelocation::_small_in_place_count;
-size_t                      ZStatRelocation::_medium_in_place_count;
+ZStatRelocation::ZStatRelocation() :
+    _selector_stats(),
+    _forwarding_usage(),
+    _small_selected(),
+    _small_in_place_count(),
+    _medium_selected(),
+    _medium_in_place_count() {
+}
 
-void ZStatRelocation::set_at_select_relocation_set(const ZRelocationSetSelectorStats& selector_stats) {
+void ZStatRelocation::at_select_relocation_set(const ZRelocationSetSelectorStats& selector_stats) {
   _selector_stats = selector_stats;
 }
 
-void ZStatRelocation::set_at_install_relocation_set(size_t forwarding_usage) {
+void ZStatRelocation::at_install_relocation_set(size_t forwarding_usage) {
   _forwarding_usage = forwarding_usage;
 }
 
-void ZStatRelocation::set_at_relocate_end(size_t small_in_place_count, size_t medium_in_place_count) {
+void ZStatRelocation::at_relocate_end(size_t small_in_place_count, size_t medium_in_place_count) {
   _small_in_place_count = small_in_place_count;
   _medium_in_place_count = medium_in_place_count;
 }
 
-void ZStatRelocation::print(const char* name,
-                            const ZRelocationSetSelectorGroupStats& selector_group,
-                            size_t in_place_count) {
-  log_info(gc, reloc)("%s Pages: " SIZE_FORMAT " / " SIZE_FORMAT "M, Empty: " SIZE_FORMAT "M, "
-                      "Relocated: " SIZE_FORMAT "M, In-Place: " SIZE_FORMAT,
-                      name,
-                      selector_group.npages(),
-                      selector_group.total() / M,
-                      selector_group.empty() / M,
-                      selector_group.relocate() / M,
-                      in_place_count);
+void ZStatRelocation::print_page_summary() {
+  LogTarget(Info, gc, reloc) lt;
+
+  if (!_selector_stats.has_relocatable_pages() || !lt.is_enabled()) {
+    // Nothing to log or logging not enabled.
+    return;
+  }
+
+  // Zero initialize
+  ZStatRelocationSummary small_summary{};
+  ZStatRelocationSummary medium_summary{};
+  ZStatRelocationSummary large_summary{};
+
+  auto account_page_size = [&](ZStatRelocationSummary& summary, const ZRelocationSetSelectorGroupStats& stats) {
+    summary.npages_candidates += stats.npages_candidates();
+    summary.total += stats.total();
+    summary.empty += stats.empty();
+    summary.npages_selected += stats.npages_selected();
+    summary.relocate += stats.relocate();
+  };
+
+  for (uint i = 0; i <= ZPageAgeMax; ++i) {
+    const ZPageAge age = static_cast<ZPageAge>(i);
+
+    account_page_size(small_summary, _selector_stats.small(age));
+    account_page_size(medium_summary, _selector_stats.medium(age));
+    account_page_size(large_summary, _selector_stats.large(age));
+  }
+
+  ZStatTablePrinter pages(20, 12);
+  lt.print("%s", pages()
+           .fill()
+           .right("Candidates")
+           .right("Selected")
+           .right("In-Place")
+           .right("Size")
+           .right("Empty")
+           .right("Relocated")
+           .end());
+
+  auto print_summary = [&](const char* name, ZStatRelocationSummary& summary, size_t in_place_count) {
+    lt.print("%s", pages()
+             .left("%s Pages:", name)
+             .right("%zu", summary.npages_candidates)
+             .right("%zu", summary.npages_selected)
+             .right("%zu", in_place_count)
+             .right("%zuM", summary.total / M)
+             .right("%zuM", summary.empty / M)
+             .right("%zuM", summary.relocate /M)
+             .end());
+  };
+
+  print_summary("Small", small_summary, _small_in_place_count);
+  if (ZPageSizeMedium != 0) {
+    print_summary("Medium", medium_summary, _medium_in_place_count);
+  }
+  print_summary("Large", large_summary, 0 /* in_place_count */);
+
+  lt.print("Forwarding Usage: " SIZE_FORMAT "M", _forwarding_usage / M);
 }
 
-void ZStatRelocation::print() {
-  print("Small", _selector_stats.small(), _small_in_place_count);
-  if (ZPageSizeMedium != 0) {
-    print("Medium", _selector_stats.medium(), _medium_in_place_count);
+void ZStatRelocation::print_age_table() {
+  LogTarget(Info, gc, reloc) lt;
+  if (!_selector_stats.has_relocatable_pages() || !lt.is_enabled()) {
+    // Nothing to log or logging not enabled.
+    return;
   }
-  print("Large", _selector_stats.large(), 0 /* in_place_count */);
 
-  log_info(gc, reloc)("Forwarding Usage: " SIZE_FORMAT "M", _forwarding_usage / M);
+  ZStatTablePrinter age_table(11, 18);
+  lt.print("Age Table:");
+  lt.print("%s", age_table()
+           .fill()
+           .center("Live")
+           .center("Garbage")
+           .center("Small")
+           .center("Medium")
+           .center("Large")
+           .end());
+
+  size_t live[ZPageAgeMax + 1] = {};
+  size_t total[ZPageAgeMax + 1] = {};
+
+  uint oldest_none_empty_age = 0;
+
+  for (uint i = 0; i <= ZPageAgeMax; ++i) {
+    ZPageAge age = static_cast<ZPageAge>(i);
+    auto summarize_pages = [&](const ZRelocationSetSelectorGroupStats& stats) {
+      live[i] += stats.live();
+      total[i] += stats.total();
+    };
+
+    summarize_pages(_selector_stats.small(age));
+    summarize_pages(_selector_stats.medium(age));
+    summarize_pages(_selector_stats.large(age));
+
+    if (total[i] != 0) {
+      oldest_none_empty_age = i;
+    }
+  }
+
+  for (uint i = 0; i <= oldest_none_empty_age; ++i) {
+    ZPageAge age = static_cast<ZPageAge>(i);
+
+    FormatBuffer<> age_str("");
+    if (age == ZPageAge::eden) {
+      age_str.append("Eden");
+    } else if (age != ZPageAge::old) {
+      age_str.append("Survivor %d", i);
+    }
+
+    auto create_age_table = [&]() {
+      if (live[i] == 0) {
+        return age_table()
+              .left("%s", age_str.buffer())
+              .left(ZTABLE_ARGS_NA);
+      } else {
+        return age_table()
+              .left("%s", age_str.buffer())
+              .left(ZTABLE_ARGS(live[i]));
+      }
+    };
+
+    lt.print("%s", create_age_table()
+              .left(ZTABLE_ARGS(total[i] - live[i]))
+              .left(SIZE_FORMAT_W(7) " / " SIZE_FORMAT,
+                    _selector_stats.small(age).npages_candidates(),
+                    _selector_stats.small(age).npages_selected())
+              .left(SIZE_FORMAT_W(7) " / " SIZE_FORMAT,
+                    _selector_stats.medium(age).npages_candidates(),
+                    _selector_stats.medium(age).npages_selected())
+              .left(SIZE_FORMAT_W(7) " / " SIZE_FORMAT,
+                    _selector_stats.large(age).npages_candidates(),
+                    _selector_stats.large(age).npages_selected())
+              .end());
+  }
 }
 
 //
@@ -1271,7 +1633,7 @@ void ZStatNMethods::print() {
 // Stat metaspace
 //
 void ZStatMetaspace::print() {
-  MetaspaceCombinedStats stats = MetaspaceUtils::get_combined_statistics();
+  const MetaspaceCombinedStats stats = MetaspaceUtils::get_combined_statistics();
   log_info(gc, metaspace)("Metaspace: "
                           SIZE_FORMAT "M used, "
                           SIZE_FORMAT "M committed, " SIZE_FORMAT "M reserved",
@@ -1310,100 +1672,159 @@ void ZStatReferences::set_phantom(size_t encountered, size_t discovered, size_t
   set(&_phantom, encountered, discovered, enqueued);
 }
 
-void ZStatReferences::print(const char* name, const ZStatReferences::ZCount& ref) {
-  log_info(gc, ref)("%s: "
-                    SIZE_FORMAT " encountered, "
-                    SIZE_FORMAT " discovered, "
-                    SIZE_FORMAT " enqueued",
-                    name,
-                    ref.encountered,
-                    ref.discovered,
-                    ref.enqueued);
-}
-
 void ZStatReferences::print() {
-  print("Soft", _soft);
-  print("Weak", _weak);
-  print("Final", _final);
-  print("Phantom", _phantom);
+  LogTarget(Info, gc, ref) lt;
+  if (!lt.is_enabled()) {
+    // Nothing to log
+    return;
+  }
+
+  ZStatTablePrinter refs(20, 12);
+  lt.print("%s", refs()
+           .fill()
+           .right("Encountered")
+           .right("Discovered")
+           .right("Enqueued")
+           .end());
+
+  auto ref_print = [&] (const char* name, const ZStatReferences::ZCount& ref) {
+    lt.print("%s", refs()
+             .left("%s References:", name)
+             .right("%zu", ref.encountered)
+             .right("%zu", ref.discovered)
+             .right("%zu", ref.enqueued)
+             .end());
+  };
+
+  ref_print("Soft", _soft);
+  ref_print("Weak", _weak);
+  ref_print("Final", _final);
+  ref_print("Phantom", _phantom);
 }
 
 //
 // Stat heap
 //
-ZStatHeap::ZAtInitialize ZStatHeap::_at_initialize;
-ZStatHeap::ZAtMarkStart ZStatHeap::_at_mark_start;
-ZStatHeap::ZAtMarkEnd ZStatHeap::_at_mark_end;
-ZStatHeap::ZAtRelocateStart ZStatHeap::_at_relocate_start;
-ZStatHeap::ZAtRelocateEnd ZStatHeap::_at_relocate_end;
 
-size_t ZStatHeap::capacity_high() {
+ZStatHeap::ZStatHeap() :
+    _stat_lock(),
+    _at_collection_start(),
+    _at_mark_start(),
+    _at_mark_end(),
+    _at_relocate_start(),
+    _at_relocate_end(),
+    _reclaimed_bytes(0.7 /* alpha */) {
+}
+
+ZStatHeap::ZAtInitialize ZStatHeap::_at_initialize;
+
+size_t ZStatHeap::capacity_high() const {
   return MAX4(_at_mark_start.capacity,
               _at_mark_end.capacity,
               _at_relocate_start.capacity,
               _at_relocate_end.capacity);
 }
 
-size_t ZStatHeap::capacity_low() {
+size_t ZStatHeap::capacity_low() const {
   return MIN4(_at_mark_start.capacity,
               _at_mark_end.capacity,
               _at_relocate_start.capacity,
               _at_relocate_end.capacity);
 }
 
-size_t ZStatHeap::free(size_t used) {
+size_t ZStatHeap::free(size_t used) const {
   return _at_initialize.max_capacity - used;
 }
 
-size_t ZStatHeap::allocated(size_t used, size_t reclaimed) {
+size_t ZStatHeap::mutator_allocated(size_t used_generation, size_t freed, size_t relocated) const {
   // The amount of allocated memory between point A and B is used(B) - used(A).
   // However, we might also have reclaimed memory between point A and B. This
   // means the current amount of used memory must be incremented by the amount
   // reclaimed, so that used(B) represents the amount of used memory we would
   // have had if we had not reclaimed anything.
-  return (used + reclaimed) - _at_mark_start.used;
+  const size_t used_generation_delta = used_generation - _at_mark_start.used_generation;
+  return  used_generation_delta + freed - relocated;
 }
 
-size_t ZStatHeap::garbage(size_t reclaimed) {
-  return _at_mark_end.garbage - reclaimed;
+size_t ZStatHeap::garbage(size_t freed, size_t relocated, size_t promoted) const {
+  return _at_mark_end.garbage - (freed - promoted - relocated);
 }
 
-void ZStatHeap::set_at_initialize(const ZPageAllocatorStats& stats) {
-  _at_initialize.min_capacity = stats.min_capacity();
-  _at_initialize.max_capacity = stats.max_capacity();
+size_t ZStatHeap::reclaimed(size_t freed, size_t relocated, size_t promoted) const {
+  return freed - relocated - promoted;
 }
 
-void ZStatHeap::set_at_mark_start(const ZPageAllocatorStats& stats) {
+void ZStatHeap::at_initialize(size_t min_capacity, size_t max_capacity) {
+  ZLocker<ZLock> locker(&_stat_lock);
+
+  _at_initialize.min_capacity = min_capacity;
+  _at_initialize.max_capacity = max_capacity;
+}
+
+void ZStatHeap::at_collection_start(const ZPageAllocatorStats& stats) {
+  ZLocker<ZLock> locker(&_stat_lock);
+
+  _at_collection_start.soft_max_capacity = stats.soft_max_capacity();
+  _at_collection_start.capacity = stats.capacity();
+  _at_collection_start.free = free(stats.used());
+  _at_collection_start.used = stats.used();
+  _at_collection_start.used_generation = stats.used_generation();
+}
+
+void ZStatHeap::at_mark_start(const ZPageAllocatorStats& stats) {
+  ZLocker<ZLock> locker(&_stat_lock);
+
   _at_mark_start.soft_max_capacity = stats.soft_max_capacity();
   _at_mark_start.capacity = stats.capacity();
   _at_mark_start.free = free(stats.used());
   _at_mark_start.used = stats.used();
+  _at_mark_start.used_generation = stats.used_generation();
+  _at_mark_start.allocation_stalls = stats.allocation_stalls();
 }
 
-void ZStatHeap::set_at_mark_end(const ZPageAllocatorStats& stats) {
+void ZStatHeap::at_mark_end(const ZPageAllocatorStats& stats) {
+  ZLocker<ZLock> locker(&_stat_lock);
+
   _at_mark_end.capacity = stats.capacity();
   _at_mark_end.free = free(stats.used());
   _at_mark_end.used = stats.used();
-  _at_mark_end.allocated = allocated(stats.used(), 0 /* reclaimed */);
+  _at_mark_end.used_generation = stats.used_generation();
+  _at_mark_end.mutator_allocated = mutator_allocated(stats.used_generation(), 0 /* reclaimed */, 0 /* relocated */);
+  _at_mark_end.allocation_stalls = stats.allocation_stalls();
 }
 
-void ZStatHeap::set_at_select_relocation_set(const ZRelocationSetSelectorStats& stats) {
-  const size_t live = stats.small().live() + stats.medium().live() + stats.large().live();
+void ZStatHeap::at_select_relocation_set(const ZRelocationSetSelectorStats& stats) {
+  ZLocker<ZLock> locker(&_stat_lock);
+
+  size_t live = 0;
+  for (uint i = 0; i <= ZPageAgeMax; ++i) {
+    const ZPageAge age = static_cast<ZPageAge>(i);
+    live += stats.small(age).live() + stats.medium(age).live() + stats.large(age).live();
+  }
   _at_mark_end.live = live;
-  _at_mark_end.garbage = _at_mark_start.used - live;
+  _at_mark_end.garbage = _at_mark_start.used_generation - live;
 }
 
-void ZStatHeap::set_at_relocate_start(const ZPageAllocatorStats& stats) {
+void ZStatHeap::at_relocate_start(const ZPageAllocatorStats& stats) {
+  ZLocker<ZLock> locker(&_stat_lock);
+
+  assert(stats.compacted() == 0, "Nothing should have been compacted");
+
   _at_relocate_start.capacity = stats.capacity();
   _at_relocate_start.free = free(stats.used());
   _at_relocate_start.used = stats.used();
-  _at_relocate_start.allocated = allocated(stats.used(), stats.reclaimed());
-  _at_relocate_start.garbage = garbage(stats.reclaimed());
-  _at_relocate_start.reclaimed = stats.reclaimed();
+  _at_relocate_start.used_generation = stats.used_generation();
+  _at_relocate_start.live = _at_mark_end.live - stats.promoted();
+  _at_relocate_start.garbage = garbage(stats.freed(), stats.compacted(), stats.promoted());
+  _at_relocate_start.mutator_allocated = mutator_allocated(stats.used_generation(), stats.freed(), stats.compacted());
+  _at_relocate_start.reclaimed = reclaimed(stats.freed(), stats.compacted(), stats.promoted());
+  _at_relocate_start.promoted = stats.promoted();
+  _at_relocate_start.compacted = stats.compacted();
+  _at_relocate_start.allocation_stalls = stats.allocation_stalls();
 }
 
-void ZStatHeap::set_at_relocate_end(const ZPageAllocatorStats& stats, size_t non_worker_relocated) {
-  const size_t reclaimed = stats.reclaimed() - MIN2(non_worker_relocated, stats.reclaimed());
+void ZStatHeap::at_relocate_end(const ZPageAllocatorStats& stats, bool record_stats) {
+  ZLocker<ZLock> locker(&_stat_lock);
 
   _at_relocate_end.capacity = stats.capacity();
   _at_relocate_end.capacity_high = capacity_high();
@@ -1414,24 +1835,87 @@ void ZStatHeap::set_at_relocate_end(const ZPageAllocatorStats& stats, size_t non
   _at_relocate_end.used = stats.used();
   _at_relocate_end.used_high = stats.used_high();
   _at_relocate_end.used_low = stats.used_low();
-  _at_relocate_end.allocated = allocated(stats.used(), reclaimed);
-  _at_relocate_end.garbage = garbage(reclaimed);
-  _at_relocate_end.reclaimed = reclaimed;
+  _at_relocate_end.used_generation = stats.used_generation();
+  _at_relocate_end.live = _at_mark_end.live - stats.promoted();
+  _at_relocate_end.garbage = garbage(stats.freed(), stats.compacted(), stats.promoted());
+  _at_relocate_end.mutator_allocated = mutator_allocated(stats.used_generation(), stats.freed(), stats.compacted());
+  _at_relocate_end.reclaimed = reclaimed(stats.freed(), stats.compacted(), stats.promoted());
+  _at_relocate_end.promoted = stats.promoted();
+  _at_relocate_end.compacted = stats.compacted();
+  _at_relocate_end.allocation_stalls = stats.allocation_stalls();
+
+  if (record_stats) {
+    _reclaimed_bytes.add(_at_relocate_end.reclaimed);
+  }
+}
+
+size_t ZStatHeap::reclaimed_avg() {
+  return _reclaimed_bytes.davg();
 }
 
 size_t ZStatHeap::max_capacity() {
   return _at_initialize.max_capacity;
 }
 
-size_t ZStatHeap::used_at_mark_start() {
+size_t ZStatHeap::used_at_collection_start() const {
+  return _at_collection_start.used;
+}
+
+size_t ZStatHeap::used_at_mark_start() const {
   return _at_mark_start.used;
 }
 
-size_t ZStatHeap::used_at_relocate_end() {
+size_t ZStatHeap::used_generation_at_mark_start() const {
+  return _at_mark_start.used_generation;
+}
+
+size_t ZStatHeap::live_at_mark_end() const {
+  return _at_mark_end.live;
+}
+
+size_t ZStatHeap::allocated_at_mark_end() const {
+  return _at_mark_end.mutator_allocated;
+}
+
+size_t ZStatHeap::garbage_at_mark_end() const {
+  return _at_mark_end.garbage;
+}
+
+size_t ZStatHeap::used_at_relocate_end() const {
   return _at_relocate_end.used;
 }
 
-void ZStatHeap::print() {
+size_t ZStatHeap::used_at_collection_end() const {
+  return used_at_relocate_end();
+}
+
+size_t ZStatHeap::stalls_at_mark_start() const {
+  return _at_mark_start.allocation_stalls;
+}
+
+size_t ZStatHeap::stalls_at_mark_end() const {
+  return _at_mark_end.allocation_stalls;
+}
+
+size_t ZStatHeap::stalls_at_relocate_start() const {
+  return _at_relocate_start.allocation_stalls;
+}
+
+size_t ZStatHeap::stalls_at_relocate_end() const {
+  return _at_relocate_end.allocation_stalls;
+}
+
+ZStatHeapStats ZStatHeap::stats() {
+  ZLocker<ZLock> locker(&_stat_lock);
+
+  return {
+    live_at_mark_end(),
+    used_at_relocate_end(),
+    reclaimed_avg()
+  };
+}
+
+void ZStatHeap::print(const ZGeneration* generation) const {
   log_info(gc, heap)("Min Capacity: "
                      ZSIZE_FMT, ZSIZE_ARGS(_at_initialize.min_capacity));
   log_info(gc, heap)("Max Capacity: "
@@ -1439,8 +1923,9 @@ void ZStatHeap::print() {
   log_info(gc, heap)("Soft Max Capacity: "
                      ZSIZE_FMT, ZSIZE_ARGS(_at_mark_start.soft_max_capacity));
 
-  ZStatTablePrinter table(10, 18);
-  log_info(gc, heap)("%s", table()
+  log_info(gc, heap)("Heap Statistics:");
+  ZStatTablePrinter heap_table(10, 18);
+  log_info(gc, heap)("%s", heap_table()
                      .fill()
                      .center("Mark Start")
                      .center("Mark End")
@@ -1449,7 +1934,7 @@ void ZStatHeap::print() {
                      .center("High")
                      .center("Low")
                      .end());
-  log_info(gc, heap)("%s", table()
+  log_info(gc, heap)("%s", heap_table()
                      .right("Capacity:")
                      .left(ZTABLE_ARGS(_at_mark_start.capacity))
                      .left(ZTABLE_ARGS(_at_mark_end.capacity))
@@ -1458,7 +1943,7 @@ void ZStatHeap::print() {
                      .left(ZTABLE_ARGS(_at_relocate_end.capacity_high))
                      .left(ZTABLE_ARGS(_at_relocate_end.capacity_low))
                      .end());
-  log_info(gc, heap)("%s", table()
+  log_info(gc, heap)("%s", heap_table()
                      .right("Free:")
                      .left(ZTABLE_ARGS(_at_mark_start.free))
                      .left(ZTABLE_ARGS(_at_mark_end.free))
@@ -1467,7 +1952,7 @@ void ZStatHeap::print() {
                      .left(ZTABLE_ARGS(_at_relocate_end.free_high))
                      .left(ZTABLE_ARGS(_at_relocate_end.free_low))
                      .end());
-  log_info(gc, heap)("%s", table()
+  log_info(gc, heap)("%s", heap_table()
                      .right("Used:")
                      .left(ZTABLE_ARGS(_at_mark_start.used))
                      .left(ZTABLE_ARGS(_at_mark_end.used))
@@ -1476,40 +1961,83 @@ void ZStatHeap::print() {
                      .left(ZTABLE_ARGS(_at_relocate_end.used_high))
                      .left(ZTABLE_ARGS(_at_relocate_end.used_low))
                      .end());
-  log_info(gc, heap)("%s", table()
+
+  log_info(gc, heap)("%s Generation Statistics:", generation->is_young() ? "Young" : "Old");
+  ZStatTablePrinter gen_table(10, 18);
+  log_info(gc, heap)("%s", gen_table()
+                     .fill()
+                     .center("Mark Start")
+                     .center("Mark End")
+                     .center("Relocate Start")
+                     .center("Relocate End")
+                     .end());
+  log_info(gc, heap)("%s", gen_table()
+                     .right("Used:")
+                     .left(ZTABLE_ARGS(_at_mark_start.used_generation))
+                     .left(ZTABLE_ARGS(_at_mark_end.used_generation))
+                     .left(ZTABLE_ARGS(_at_relocate_start.used_generation))
+                     .left(ZTABLE_ARGS(_at_relocate_end.used_generation))
+                     .end());
+  log_info(gc, heap)("%s", gen_table()
                      .right("Live:")
                      .left(ZTABLE_ARGS_NA)
                      .left(ZTABLE_ARGS(_at_mark_end.live))
-                     .left(ZTABLE_ARGS(_at_mark_end.live /* Same as at mark end */))
-                     .left(ZTABLE_ARGS(_at_mark_end.live /* Same as at mark end */))
-                     .left(ZTABLE_ARGS_NA)
-                     .left(ZTABLE_ARGS_NA)
+                     .left(ZTABLE_ARGS(_at_relocate_start.live))
+                     .left(ZTABLE_ARGS(_at_relocate_end.live))
                      .end());
-  log_info(gc, heap)("%s", table()
-                     .right("Allocated:")
-                     .left(ZTABLE_ARGS_NA)
-                     .left(ZTABLE_ARGS(_at_mark_end.allocated))
-                     .left(ZTABLE_ARGS(_at_relocate_start.allocated))
-                     .left(ZTABLE_ARGS(_at_relocate_end.allocated))
-                     .left(ZTABLE_ARGS_NA)
-                     .left(ZTABLE_ARGS_NA)
-                     .end());
-  log_info(gc, heap)("%s", table()
+  log_info(gc, heap)("%s", gen_table()
                      .right("Garbage:")
                      .left(ZTABLE_ARGS_NA)
                      .left(ZTABLE_ARGS(_at_mark_end.garbage))
                      .left(ZTABLE_ARGS(_at_relocate_start.garbage))
                      .left(ZTABLE_ARGS(_at_relocate_end.garbage))
-                     .left(ZTABLE_ARGS_NA)
-                     .left(ZTABLE_ARGS_NA)
                      .end());
-  log_info(gc, heap)("%s", table()
+  log_info(gc, heap)("%s", gen_table()
+                     .right("Allocated:")
+                     .left(ZTABLE_ARGS_NA)
+                     .left(ZTABLE_ARGS(_at_mark_end.mutator_allocated))
+                     .left(ZTABLE_ARGS(_at_relocate_start.mutator_allocated))
+                     .left(ZTABLE_ARGS(_at_relocate_end.mutator_allocated))
+                     .end());
+  log_info(gc, heap)("%s", gen_table()
                      .right("Reclaimed:")
                      .left(ZTABLE_ARGS_NA)
                      .left(ZTABLE_ARGS_NA)
                      .left(ZTABLE_ARGS(_at_relocate_start.reclaimed))
                      .left(ZTABLE_ARGS(_at_relocate_end.reclaimed))
+                     .end());
+  if (generation->is_young()) {
+    log_info(gc, heap)("%s", gen_table()
+                       .right("Promoted:")
+                       .left(ZTABLE_ARGS_NA)
+                       .left(ZTABLE_ARGS_NA)
+                       .left(ZTABLE_ARGS(_at_relocate_start.promoted))
+                       .left(ZTABLE_ARGS(_at_relocate_end.promoted))
+                       .end());
+  }
+  log_info(gc, heap)("%s", gen_table()
+                     .right("Compacted:")
                      .left(ZTABLE_ARGS_NA)
                      .left(ZTABLE_ARGS_NA)
+                     .left(ZTABLE_ARGS_NA)
+                     .left(ZTABLE_ARGS(_at_relocate_end.compacted))
+                     .end());
+}
+
+void ZStatHeap::print_stalls() const {
+  ZStatTablePrinter stall_table(20, 16);
+  log_info(gc, alloc)("%s", stall_table()
+                     .fill()
+                     .center("Mark Start")
+                     .center("Mark End")
+                     .center("Relocate Start")
+                     .center("Relocate End")
+                     .end());
+  log_info(gc, alloc)("%s", stall_table()
+                     .left("%s", "Allocation Stalls:")
+                     .center("%zu", _at_mark_start.allocation_stalls)
+                     .center("%zu", _at_mark_end.allocation_stalls)
+                     .center("%zu", _at_relocate_start.allocation_stalls)
+                     .center("%zu", _at_relocate_end.allocation_stalls)
                      .end());
 }
diff --git a/src/hotspot/share/gc/z/zStat.hpp b/src/hotspot/share/gc/z/zStat.hpp
index 65b993811ad..5b129c26fc9 100644
--- a/src/hotspot/share/gc/z/zStat.hpp
+++ b/src/hotspot/share/gc/z/zStat.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,22 +24,28 @@
 #ifndef SHARE_GC_Z_ZSTAT_HPP
 #define SHARE_GC_Z_ZSTAT_HPP
 
-#include "gc/shared/concurrentGCThread.hpp"
 #include "gc/shared/gcCause.hpp"
 #include "gc/shared/gcTimer.hpp"
+#include "gc/z/zGenerationId.hpp"
+#include "gc/z/zLock.hpp"
 #include "gc/z/zMetronome.hpp"
+#include "gc/z/zRelocationSetSelector.hpp"
+#include "gc/z/zThread.hpp"
+#include "gc/z/zTracer.hpp"
 #include "logging/logHandle.hpp"
 #include "memory/allocation.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/numberSeq.hpp"
 #include "utilities/ticks.hpp"
 
+class GCTracer;
+class ZGeneration;
 class ZPage;
 class ZPageAllocatorStats;
 class ZRelocationSetSelectorGroupStats;
-class ZRelocationSetSelectorStats;
 class ZStatSampler;
 class ZStatSamplerHistory;
+class ZStatWorkers;
 struct ZStatCounterData;
 struct ZStatSamplerData;
 
@@ -204,9 +210,6 @@ public:
 // Stat phases
 //
 class ZStatPhase {
-private:
-  static ConcurrentGCTimer _timer;
-
 protected:
   const ZStatSampler _sampler;
 
@@ -216,20 +219,39 @@ protected:
   void log_end(LogTargetHandle log, const Tickspan& duration, bool thread = false) const;
 
 public:
-  static ConcurrentGCTimer* timer();
-
   const char* name() const;
 
-  virtual void register_start(const Ticks& start) const = 0;
-  virtual void register_end(const Ticks& start, const Ticks& end) const = 0;
+  virtual void register_start(ConcurrentGCTimer* timer, const Ticks& start) const = 0;
+  virtual void register_end(ConcurrentGCTimer* timer, const Ticks& start, const Ticks& end) const = 0;
 };
 
-class ZStatPhaseCycle : public ZStatPhase {
-public:
-  ZStatPhaseCycle(const char* name);
+class ZStatPhaseCollection : public ZStatPhase {
+private:
+  const bool _minor;
 
-  virtual void register_start(const Ticks& start) const;
-  virtual void register_end(const Ticks& start, const Ticks& end) const;
+  GCTracer* jfr_tracer() const;
+
+  void set_used_at_start(size_t used) const;
+  size_t used_at_start() const;
+
+public:
+  ZStatPhaseCollection(const char* name, bool minor);
+
+  virtual void register_start(ConcurrentGCTimer* timer, const Ticks& start) const;
+  virtual void register_end(ConcurrentGCTimer* timer, const Ticks& start, const Ticks& end) const;
+};
+
+class ZStatPhaseGeneration : public ZStatPhase {
+private:
+  const ZGenerationId _id;
+
+  ZGenerationTracer* jfr_tracer() const;
+
+public:
+  ZStatPhaseGeneration(const char* name, ZGenerationId id);
+
+  virtual void register_start(ConcurrentGCTimer* timer, const Ticks& start) const;
+  virtual void register_end(ConcurrentGCTimer* timer, const Ticks& start, const Ticks& end) const;
 };
 
 class ZStatPhasePause : public ZStatPhase {
@@ -237,28 +259,28 @@ private:
   static Tickspan _max; // Max pause time
 
 public:
-  ZStatPhasePause(const char* name);
+  ZStatPhasePause(const char* name, ZGenerationId id);
 
   static const Tickspan& max();
 
-  virtual void register_start(const Ticks& start) const;
-  virtual void register_end(const Ticks& start, const Ticks& end) const;
+  virtual void register_start(ConcurrentGCTimer* timer, const Ticks& start) const;
+  virtual void register_end(ConcurrentGCTimer* timer, const Ticks& start, const Ticks& end) const;
 };
 
 class ZStatPhaseConcurrent : public ZStatPhase {
 public:
-  ZStatPhaseConcurrent(const char* name);
+  ZStatPhaseConcurrent(const char* name, ZGenerationId id);
 
-  virtual void register_start(const Ticks& start) const;
-  virtual void register_end(const Ticks& start, const Ticks& end) const;
+  virtual void register_start(ConcurrentGCTimer* timer, const Ticks& start) const;
+  virtual void register_end(ConcurrentGCTimer* timer, const Ticks& start, const Ticks& end) const;
 };
 
 class ZStatSubPhase : public ZStatPhase {
 public:
-  ZStatSubPhase(const char* name);
+  ZStatSubPhase(const char* name, ZGenerationId id);
 
-  virtual void register_start(const Ticks& start) const;
-  virtual void register_end(const Ticks& start, const Ticks& end) const;
+  virtual void register_start(ConcurrentGCTimer* timer, const Ticks& start) const;
+  virtual void register_end(ConcurrentGCTimer* timer, const Ticks& start, const Ticks& end) const;
 };
 
 class ZStatCriticalPhase : public ZStatPhase {
@@ -269,55 +291,56 @@ private:
 public:
   ZStatCriticalPhase(const char* name, bool verbose = true);
 
-  virtual void register_start(const Ticks& start) const;
-  virtual void register_end(const Ticks& start, const Ticks& end) const;
+  virtual void register_start(ConcurrentGCTimer* timer, const Ticks& start) const;
+  virtual void register_end(ConcurrentGCTimer* timer, const Ticks& start, const Ticks& end) const;
 };
 
 //
 // Stat timer
 //
-class ZStatTimerDisable : public StackObj {
-private:
-  static THREAD_LOCAL uint32_t _active;
-
-public:
-  ZStatTimerDisable() {
-    _active++;
-  }
-
-  ~ZStatTimerDisable() {
-    _active--;
-  }
-
-  static bool is_active() {
-    return _active > 0;
-  }
-};
-
 class ZStatTimer : public StackObj {
 private:
-  const bool        _enabled;
-  const ZStatPhase& _phase;
-  const Ticks       _start;
+  ConcurrentGCTimer* const _gc_timer;
+  const ZStatPhase&        _phase;
+  const Ticks              _start;
 
 public:
-  ZStatTimer(const ZStatPhase& phase) :
-      _enabled(!ZStatTimerDisable::is_active()),
+  ZStatTimer(const ZStatPhase& phase, ConcurrentGCTimer* gc_timer) :
+      _gc_timer(gc_timer),
       _phase(phase),
       _start(Ticks::now()) {
-    if (_enabled) {
-      _phase.register_start(_start);
-    }
+    _phase.register_start(_gc_timer, _start);
+  }
+
+  ZStatTimer(const ZStatSubPhase& phase) :
+      ZStatTimer(phase, nullptr /* timer */) {
+  }
+
+  ZStatTimer(const ZStatCriticalPhase& phase) :
+      ZStatTimer(phase, nullptr /* timer */) {
   }
 
   ~ZStatTimer() {
-    if (_enabled) {
-      const Ticks end = Ticks::now();
-      _phase.register_end(_start, end);
-    }
+    const Ticks end = Ticks::now();
+    _phase.register_end(_gc_timer, _start, end);
   }
 };
 
+class ZStatTimerYoung : public ZStatTimer {
+public:
+  ZStatTimerYoung(const ZStatPhase& phase);
+};
+
+class ZStatTimerOld : public ZStatTimer {
+public:
+  ZStatTimerOld(const ZStatPhase& phase);
+};
+
+class ZStatTimerWorker : public ZStatTimer {
+public:
+  ZStatTimerWorker(const ZStatPhase& phase);
+};
+
 //
 // Stat sample/increment
 //
@@ -325,30 +348,40 @@ void ZStatSample(const ZStatSampler& sampler, uint64_t value);
 void ZStatInc(const ZStatCounter& counter, uint64_t increment = 1);
 void ZStatInc(const ZStatUnsampledCounter& counter, uint64_t increment = 1);
 
+struct ZStatMutatorAllocRateStats {
+  double _avg;
+  double _predict;
+  double _sd;
+};
+
 //
-// Stat allocation rate
+// Stat mutator allocation rate
 //
-class ZStatAllocRate : public AllStatic {
+class ZStatMutatorAllocRate : public AllStatic {
 private:
-  static const ZStatUnsampledCounter _counter;
-  static TruncatedSeq                _samples;
-  static TruncatedSeq                _rate;
+  static ZLock*          _stat_lock;
+  static jlong           _last_sample_time;
+  static volatile size_t _sampling_granule;
+  static volatile size_t _allocated_since_sample;
+  static TruncatedSeq    _samples_time;
+  static TruncatedSeq    _samples_bytes;
+  static TruncatedSeq    _rate;
+
+  static void update_sampling_granule();
 
 public:
-  static const uint64_t sample_hz = 10;
-
   static const ZStatUnsampledCounter& counter();
-  static uint64_t sample_and_reset();
+  static void sample_allocation(size_t allocation_bytes);
 
-  static double predict();
-  static double avg();
-  static double sd();
+  static void initialize();
+
+  static ZStatMutatorAllocRateStats stats();
 };
 
 //
 // Stat thread
 //
-class ZStat : public ConcurrentGCThread {
+class ZStat : public ZThread {
 private:
   static const uint64_t sample_hz = 1;
 
@@ -359,54 +392,89 @@ private:
   void print(LogTargetHandle log, const ZStatSamplerHistory* history) const;
 
 protected:
-  virtual void run_service();
-  virtual void stop_service();
+  virtual void run_thread();
+  virtual void terminate();
 
 public:
   ZStat();
 };
 
+struct ZStatCycleStats {
+  bool _is_warm;
+  uint64_t _nwarmup_cycles;
+  bool _is_time_trustable;
+  double _time_since_last;
+  double _last_active_workers;
+  double _duration_since_start;
+  double _avg_cycle_interval;
+  double _avg_serial_time;
+  double _sd_serial_time;
+  double _avg_parallelizable_time;
+  double _sd_parallelizable_time;
+  double _avg_parallelizable_duration;
+  double _sd_parallelizable_duration;
+};
+
 //
 // Stat cycle
 //
-class ZStatCycle : public AllStatic {
+class ZStatCycle {
 private:
-  static uint64_t  _nwarmup_cycles;
-  static Ticks     _start_of_last;
-  static Ticks     _end_of_last;
-  static NumberSeq _serial_time;
-  static NumberSeq _parallelizable_time;
-  static uint      _last_active_workers;
+  ZLock     _stat_lock;
+  uint64_t  _nwarmup_cycles;
+  Ticks     _start_of_last;
+  Ticks     _end_of_last;
+  NumberSeq _cycle_intervals;
+  NumberSeq _serial_time;
+  NumberSeq _parallelizable_time;
+  NumberSeq _parallelizable_duration;
+  double    _last_active_workers;
+
+  bool is_warm();
+  bool is_time_trustable();
+  double last_active_workers();
+  double duration_since_start();
+  double time_since_last();
 
 public:
-  static void at_start();
-  static void at_end(GCCause::Cause cause, uint active_workers);
+  ZStatCycle();
 
-  static bool is_warm();
-  static uint64_t nwarmup_cycles();
+  void at_start();
+  void at_end(ZStatWorkers* stats_workers, bool record_stats);
 
-  static bool is_time_trustable();
-  static const AbsSeq& serial_time();
-  static const AbsSeq& parallelizable_time();
+  ZStatCycleStats stats();
+};
 
-  static uint last_active_workers();
-
-  static double time_since_last();
+struct ZStatWorkersStats {
+  double _accumulated_time;
+  double _accumulated_duration;
 };
 
 //
 // Stat workers
 //
-class ZStatWorkers : public AllStatic {
+class ZStatWorkers {
 private:
-  static Ticks    _start_of_last;
-  static Tickspan _accumulated_duration;
+  ZLock    _stat_lock;
+  uint     _active_workers;
+  Ticks    _start_of_last;
+  Tickspan _accumulated_duration;
+  Tickspan _accumulated_time;
+
+  double accumulated_duration();
+  double accumulated_time();
+  uint active_workers();
 
 public:
-  static void at_start();
-  static void at_end();
+  ZStatWorkers();
 
-  static double get_and_reset_duration();
+  void at_start(uint active_workers);
+  void at_end();
+
+  double get_and_reset_duration();
+  double get_and_reset_time();
+
+  ZStatWorkersStats stats();
 };
 
 //
@@ -420,46 +488,62 @@ public:
 //
 // Stat mark
 //
-class ZStatMark : public AllStatic {
+class ZStatMark {
 private:
-  static size_t _nstripes;
-  static size_t _nproactiveflush;
-  static size_t _nterminateflush;
-  static size_t _ntrycomplete;
-  static size_t _ncontinue;
-  static size_t _mark_stack_usage;
+  size_t _nstripes;
+  size_t _nproactiveflush;
+  size_t _nterminateflush;
+  size_t _ntrycomplete;
+  size_t _ncontinue;
+  size_t _mark_stack_usage;
 
 public:
-  static void set_at_mark_start(size_t nstripes);
-  static void set_at_mark_end(size_t nproactiveflush,
-                              size_t nterminateflush,
-                              size_t ntrycomplete,
-                              size_t ncontinue);
-  static void set_at_mark_free(size_t mark_stack_usage);
+  ZStatMark();
 
-  static void print();
+  void at_mark_start(size_t nstripes);
+  void at_mark_end(size_t nproactiveflush,
+                   size_t nterminateflush,
+                   size_t ntrycomplete,
+                   size_t ncontinue);
+  void at_mark_free(size_t mark_stack_usage);
+
+  void print();
+};
+
+struct ZStatRelocationSummary {
+  size_t npages_candidates;
+  size_t total;
+  size_t live;
+  size_t empty;
+  size_t npages_selected;
+  size_t relocate;
 };
 
 //
 // Stat relocation
 //
-class ZStatRelocation : public AllStatic {
+class ZStatRelocation {
 private:
-  static ZRelocationSetSelectorStats _selector_stats;
-  static size_t                      _forwarding_usage;
-  static size_t                      _small_in_place_count;
-  static size_t                      _medium_in_place_count;
+  ZRelocationSetSelectorStats _selector_stats;
+  size_t                      _forwarding_usage;
+  size_t                      _small_selected;
+  size_t                      _small_in_place_count;
+  size_t                      _medium_selected;
+  size_t                      _medium_in_place_count;
 
-  static void print(const char* name,
-                    const ZRelocationSetSelectorGroupStats& selector_group,
-                    size_t in_place_count);
+  void print(const char* name,
+             ZStatRelocationSummary selector_group,
+             size_t in_place_count);
 
 public:
-  static void set_at_select_relocation_set(const ZRelocationSetSelectorStats& selector_stats);
-  static void set_at_install_relocation_set(size_t forwarding_usage);
-  static void set_at_relocate_end(size_t small_in_place_count, size_t medium_in_place_count);
+  ZStatRelocation();
 
-  static void print();
+  void at_select_relocation_set(const ZRelocationSetSelectorStats& selector_stats);
+  void at_install_relocation_set(size_t forwarding_usage);
+  void at_relocate_end(size_t small_in_place_count, size_t medium_in_place_count);
+
+  void print_page_summary();
+  void print_age_table();
 };
 
 //
@@ -490,7 +574,6 @@ private:
   } _soft, _weak, _final, _phantom;
 
   static void set(ZCount* count, size_t encountered, size_t discovered, size_t enqueued);
-  static void print(const char* name, const ZCount& ref);
 
 public:
   static void set_soft(size_t encountered, size_t discovered, size_t enqueued);
@@ -501,42 +584,67 @@ public:
   static void print();
 };
 
+struct ZStatHeapStats {
+  size_t _live_at_mark_end;
+  size_t _used_at_relocate_end;
+  size_t _reclaimed_avg;
+};
+
 //
 // Stat heap
 //
-class ZStatHeap : public AllStatic {
+class ZStatHeap {
 private:
+  ZLock _stat_lock;
+
   static struct ZAtInitialize {
     size_t min_capacity;
     size_t max_capacity;
   } _at_initialize;
 
-  static struct ZAtMarkStart {
+  struct ZAtGenerationCollectionStart {
     size_t soft_max_capacity;
     size_t capacity;
     size_t free;
     size_t used;
+    size_t used_generation;
+  } _at_collection_start;
+
+  struct ZAtMarkStart {
+    size_t soft_max_capacity;
+    size_t capacity;
+    size_t free;
+    size_t used;
+    size_t used_generation;
+    size_t allocation_stalls;
   } _at_mark_start;
 
-  static struct ZAtMarkEnd {
+  struct ZAtMarkEnd {
     size_t capacity;
     size_t free;
     size_t used;
+    size_t used_generation;
     size_t live;
-    size_t allocated;
     size_t garbage;
+    size_t mutator_allocated;
+    size_t allocation_stalls;
   } _at_mark_end;
 
-  static struct ZAtRelocateStart {
+  struct ZAtRelocateStart {
     size_t capacity;
     size_t free;
     size_t used;
-    size_t allocated;
+    size_t used_generation;
+    size_t live;
     size_t garbage;
+    size_t mutator_allocated;
     size_t reclaimed;
+    size_t promoted;
+    size_t compacted;
+    size_t allocation_stalls;
   } _at_relocate_start;
 
-  static struct ZAtRelocateEnd {
+  struct ZAtRelocateEnd {
     size_t capacity;
     size_t capacity_high;
     size_t capacity_low;
@@ -546,30 +654,56 @@ private:
     size_t used;
     size_t used_high;
     size_t used_low;
-    size_t allocated;
+    size_t used_generation;
+    size_t live;
     size_t garbage;
+    size_t mutator_allocated;
     size_t reclaimed;
+    size_t promoted;
+    size_t compacted;
+    size_t allocation_stalls;
   } _at_relocate_end;
 
-  static size_t capacity_high();
-  static size_t capacity_low();
-  static size_t free(size_t used);
-  static size_t allocated(size_t used, size_t reclaimed);
-  static size_t garbage(size_t reclaimed);
+  NumberSeq _reclaimed_bytes;
+
+  size_t capacity_high() const;
+  size_t capacity_low() const;
+  size_t free(size_t used) const;
+  size_t mutator_allocated(size_t used, size_t freed, size_t relocated) const;
+  size_t garbage(size_t freed, size_t relocated, size_t promoted) const;
+  size_t reclaimed(size_t freed, size_t relocated, size_t promoted) const;
 
 public:
-  static void set_at_initialize(const ZPageAllocatorStats& stats);
-  static void set_at_mark_start(const ZPageAllocatorStats& stats);
-  static void set_at_mark_end(const ZPageAllocatorStats& stats);
-  static void set_at_select_relocation_set(const ZRelocationSetSelectorStats& stats);
-  static void set_at_relocate_start(const ZPageAllocatorStats& stats);
-  static void set_at_relocate_end(const ZPageAllocatorStats& stats, size_t non_worker_relocated);
+  ZStatHeap();
+
+  void at_initialize(size_t min_capacity, size_t max_capacity);
+  void at_collection_start(const ZPageAllocatorStats& stats);
+  void at_mark_start(const ZPageAllocatorStats& stats);
+  void at_mark_end(const ZPageAllocatorStats& stats);
+  void at_select_relocation_set(const ZRelocationSetSelectorStats& stats);
+  void at_relocate_start(const ZPageAllocatorStats& stats);
+  void at_relocate_end(const ZPageAllocatorStats& stats, bool record_stats);
 
   static size_t max_capacity();
-  static size_t used_at_mark_start();
-  static size_t used_at_relocate_end();
+  size_t used_at_collection_start() const;
+  size_t used_at_mark_start() const;
+  size_t used_generation_at_mark_start() const;
+  size_t live_at_mark_end() const;
+  size_t allocated_at_mark_end() const;
+  size_t garbage_at_mark_end() const;
+  size_t used_at_relocate_end() const;
+  size_t used_at_collection_end() const;
+  size_t stalls_at_mark_start() const;
+  size_t stalls_at_mark_end() const;
+  size_t stalls_at_relocate_start() const;
+  size_t stalls_at_relocate_end() const;
 
-  static void print();
+  size_t reclaimed_avg();
+
+  ZStatHeapStats stats();
+
+  void print(const ZGeneration* generation) const;
+  void print_stalls() const;
 };
 
 #endif // SHARE_GC_Z_ZSTAT_HPP
diff --git a/src/hotspot/share/gc/z/zStoreBarrierBuffer.cpp b/src/hotspot/share/gc/z/zStoreBarrierBuffer.cpp
new file mode 100644
index 00000000000..015b22cadaf
--- /dev/null
+++ b/src/hotspot/share/gc/z/zStoreBarrierBuffer.cpp
@@ -0,0 +1,320 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/shared/gc_globals.hpp"
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zBarrier.inline.hpp"
+#include "gc/z/zGeneration.inline.hpp"
+#include "gc/z/zStoreBarrierBuffer.inline.hpp"
+#include "gc/z/zUncoloredRoot.inline.hpp"
+#include "memory/resourceArea.hpp"
+#include "runtime/threadSMR.hpp"
+#include "utilities/ostream.hpp"
+#include "utilities/vmError.hpp"
+
+ByteSize ZStoreBarrierEntry::p_offset() {
+  return byte_offset_of(ZStoreBarrierEntry, _p);
+}
+
+ByteSize ZStoreBarrierEntry::prev_offset() {
+  return byte_offset_of(ZStoreBarrierEntry, _prev);
+}
+
+ByteSize ZStoreBarrierBuffer::buffer_offset() {
+  return byte_offset_of(ZStoreBarrierBuffer, _buffer);
+}
+
+ByteSize ZStoreBarrierBuffer::current_offset() {
+  return byte_offset_of(ZStoreBarrierBuffer, _current);
+}
+
+ZStoreBarrierBuffer::ZStoreBarrierBuffer() :
+    _buffer(),
+    _last_processed_color(),
+    _last_installed_color(),
+    _base_pointer_lock(),
+    _base_pointers(),
+    _current(ZBufferStoreBarriers ? _buffer_size_bytes : 0) {
+}
+
+void ZStoreBarrierBuffer::initialize() {
+  _last_processed_color = ZPointerStoreGoodMask;
+  _last_installed_color = ZPointerStoreGoodMask;
+}
+
+void ZStoreBarrierBuffer::clear() {
+  _current = _buffer_size_bytes;
+}
+
+bool ZStoreBarrierBuffer::is_empty() const {
+  return _current == _buffer_size_bytes;
+}
+
+void ZStoreBarrierBuffer::install_base_pointers_inner() {
+  assert(ZPointer::remap_bits(_last_installed_color) ==
+         ZPointer::remap_bits(_last_processed_color),
+         "Can't deal with two pending base pointer installations");
+
+  assert((ZPointer::remap_bits(_last_processed_color) & ZPointerRemappedYoungMask) == 0 ||
+         (ZPointer::remap_bits(_last_processed_color) & ZPointerRemappedOldMask) == 0,
+         "Should not have double bit errors");
+
+  for (int i = current(); i < (int)_buffer_length; ++i) {
+    const ZStoreBarrierEntry& entry = _buffer[i];
+    volatile zpointer* const p = entry._p;
+    const zaddress_unsafe p_unsafe = to_zaddress_unsafe((uintptr_t)p);
+
+    // Color with the last processed color
+    const zpointer ptr = ZAddress::color(p_unsafe, _last_processed_color);
+
+    // Look up the generation that thinks that this pointer is not
+    // load good and check if the page is being relocated.
+    ZGeneration* const remap_generation = ZBarrier::remap_generation(ptr);
+    ZForwarding* const forwarding = remap_generation->forwarding(p_unsafe);
+    if (forwarding != nullptr) {
+      // Page is being relocated
+      ZPage* const page = forwarding->page();
+      _base_pointers[i] = page->find_base(p);
+    } else {
+      // Page is not being relocated
+      _base_pointers[i] = zaddress_unsafe::null;
+    }
+  }
+}
+
+void ZStoreBarrierBuffer::install_base_pointers() {
+  if (!ZBufferStoreBarriers) {
+    return;
+  }
+
+  // Use a lock since both the GC and the Java thread race to install the base pointers
+  ZLocker<ZLock> locker(&_base_pointer_lock);
+
+  const bool should_install_base_pointers = ZPointer::remap_bits(_last_installed_color) != ZPointerRemapped;
+
+  if (should_install_base_pointers) {
+    install_base_pointers_inner();
+  }
+
+  // This is used as a claim mechanism to make sure that we only install the base pointers once
+  _last_installed_color = ZPointerStoreGoodMask;
+}
+
+static volatile zpointer* make_load_good(volatile zpointer* p, zaddress_unsafe p_base, uintptr_t color) {
+  assert(!is_null(p_base), "need base pointer");
+
+  // Calculate field offset before p_base is remapped
+  const uintptr_t offset = (uintptr_t)p - untype(p_base);
+
+  // Remap local-copy of base pointer
+  ZUncoloredRoot::process_no_keepalive(&p_base, color);
+
+  // Retype now that the address is known to point to the correct address
+  const zaddress p_base_remapped = safe(p_base);
+
+  assert(offset < ZUtils::object_size(p_base_remapped),
+         "wrong base object; live bits are invalid");
+
+  // Calculate remapped field address
+  const zaddress p_remapped = to_zaddress(untype(p_base_remapped) + offset);
+
+  return (volatile zpointer*)p_remapped;
+}
+
+void ZStoreBarrierBuffer::on_new_phase_relocate(int i) {
+  const uintptr_t last_remap_bits = ZPointer::remap_bits(_last_processed_color);
+  if (last_remap_bits == ZPointerRemapped) {
+    // All pointers are already remapped
+    return;
+  }
+
+  const zaddress_unsafe p_base = _base_pointers[i];
+  if (is_null(p_base)) {
+    // Page is not part of the relocation set
+    return;
+  }
+
+  ZStoreBarrierEntry& entry = _buffer[i];
+
+  // Relocate the base object and calculate the remapped p
+  entry._p = make_load_good(entry._p, p_base, _last_processed_color);
+}
+
+void ZStoreBarrierBuffer::on_new_phase_remember(int i) {
+  volatile zpointer* const p = _buffer[i]._p;
+
+  if (ZHeap::heap()->is_young(p)) {
+    // Only need remset entries for old objects
+    return;
+  }
+
+  const uintptr_t last_mark_young_bits = _last_processed_color & (ZPointerMarkedYoung0 | ZPointerMarkedYoung1);
+  const bool woke_up_in_young_mark = last_mark_young_bits != ZPointerMarkedYoung;
+
+  if (woke_up_in_young_mark) {
+    // When young mark starts we "flip" the remembered sets. The remembered
+    // sets used before the young mark start becomes read-only and used by
+    // the GC to scan for old-to-young pointers to use as marking roots.
+    //
+    // Entries in the store buffer that were added before the mark young start,
+    // were supposed to be part of the remembered sets that the GC scans.
+    // However, it is too late to add those entries at this point, so instead
+    // we perform the GC remembered set scanning up-front here.
+    ZGeneration::young()->scan_remembered_field(p);
+  } else {
+    // The remembered set wasn't flipped in this phase shift,
+    // so just add the remembered set entry.
+    ZGeneration::young()->remember(p);
+  }
+}
+
+bool ZStoreBarrierBuffer::is_old_mark() const {
+  return ZGeneration::old()->is_phase_mark();
+}
+
+bool ZStoreBarrierBuffer::stored_during_old_mark() const {
+  const uintptr_t last_mark_old_bits = _last_processed_color & (ZPointerMarkedOld0 | ZPointerMarkedOld1);
+  return last_mark_old_bits == ZPointerMarkedOld;
+}
+
+void ZStoreBarrierBuffer::on_new_phase_mark(int i) {
+  const ZStoreBarrierEntry& entry = _buffer[i];
+  const zpointer prev = entry._prev;
+
+  if (is_null_any(prev)) {
+    return;
+  }
+
+  volatile zpointer* const p = entry._p;
+
+  // Young collections can start during old collections, but not the other
+  // way around. Therefore, only old marking can see a collection phase
+  // shift (resulting in a call to this function).
+  //
+  // Stores before the marking phase started is not a part of the SATB snapshot,
+  // and therefore shouldn't be used for marking.
+  //
+  // Locations in the young generation are not part of the old marking.
+  if (is_old_mark() && stored_during_old_mark() && ZHeap::heap()->is_old(p)) {
+    const zaddress addr = ZBarrier::make_load_good(prev);
+    ZUncoloredRoot::mark_object(addr);
+  }
+}
+
+void ZStoreBarrierBuffer::on_new_phase() {
+  if (!ZBufferStoreBarriers) {
+    return;
+  }
+
+  // Install all base pointers for relocation
+  install_base_pointers();
+
+  for (int i = current(); i < (int)_buffer_length; ++i) {
+    on_new_phase_relocate(i);
+    on_new_phase_remember(i);
+    on_new_phase_mark(i);
+  }
+
+  clear();
+
+  _last_processed_color = ZPointerStoreGoodMask;
+  assert(_last_installed_color == _last_processed_color, "invariant");
+}
+
+class ZStoreBarrierBuffer::OnError : public VMErrorCallback {
+private:
+  ZStoreBarrierBuffer* _buffer;
+
+public:
+  OnError(ZStoreBarrierBuffer* buffer) :
+      _buffer(buffer) {}
+
+  virtual void call(outputStream* st) {
+    _buffer->on_error(st);
+  }
+};
+
+void ZStoreBarrierBuffer::on_error(outputStream* st) {
+  st->print_cr("ZStoreBarrierBuffer: error when flushing");
+  st->print_cr(" _last_processed_color: " PTR_FORMAT, _last_processed_color);
+  st->print_cr(" _last_installed_color: " PTR_FORMAT, _last_installed_color);
+
+  for (int i = current(); i < (int)_buffer_length; ++i) {
+    st->print_cr(" [%2d]: base: " PTR_FORMAT " p: " PTR_FORMAT " prev: " PTR_FORMAT,
+        i,
+        untype(_base_pointers[i]),
+        p2i(_buffer[i]._p),
+        untype(_buffer[i]._prev));
+  }
+}
+
+void ZStoreBarrierBuffer::flush() {
+  if (!ZBufferStoreBarriers) {
+    return;
+  }
+
+  OnError on_error(this);
+  VMErrorCallbackMark mark(&on_error);
+
+  for (int i = current(); i < (int)_buffer_length; ++i) {
+    const ZStoreBarrierEntry& entry = _buffer[i];
+    const zaddress addr = ZBarrier::make_load_good(entry._prev);
+    ZBarrier::mark_and_remember(entry._p, addr);
+  }
+
+  clear();
+}
+
+bool ZStoreBarrierBuffer::is_in(volatile zpointer* p) {
+  if (!ZBufferStoreBarriers) {
+    return false;
+  }
+
+  for (JavaThreadIteratorWithHandle jtiwh; JavaThread * const jt = jtiwh.next(); ) {
+    ZStoreBarrierBuffer* const buffer = ZThreadLocalData::store_barrier_buffer(jt);
+
+    const uintptr_t  last_remap_bits = ZPointer::remap_bits(buffer->_last_processed_color) & ZPointerRemappedMask;
+    const bool needs_remap = last_remap_bits != ZPointerRemapped;
+
+    for (int i = buffer->current(); i < (int)_buffer_length; ++i) {
+      const ZStoreBarrierEntry& entry = buffer->_buffer[i];
+      volatile zpointer* entry_p = entry._p;
+
+      // Potentially remap p
+      if (needs_remap) {
+        const zaddress_unsafe entry_p_base = buffer->_base_pointers[i];
+        if (!is_null(entry_p_base)) {
+          entry_p = make_load_good(entry_p, entry_p_base, buffer->_last_processed_color);
+        }
+      }
+
+      // Check if p matches
+      if (entry_p == p) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
diff --git a/src/hotspot/share/gc/z/zStoreBarrierBuffer.hpp b/src/hotspot/share/gc/z/zStoreBarrierBuffer.hpp
new file mode 100644
index 00000000000..f917a6c3e7b
--- /dev/null
+++ b/src/hotspot/share/gc/z/zStoreBarrierBuffer.hpp
@@ -0,0 +1,98 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZSTOREBARRIERBUFFER_HPP
+#define SHARE_GC_Z_ZSTOREBARRIERBUFFER_HPP
+
+#include "gc/z/zAddress.hpp"
+#include "gc/z/zGenerationId.hpp"
+#include "gc/z/zLock.hpp"
+#include "memory/allocation.hpp"
+#include "utilities/sizes.hpp"
+
+struct ZStoreBarrierEntry {
+  volatile zpointer* _p;
+  zpointer           _prev;
+
+  static ByteSize p_offset();
+  static ByteSize prev_offset();
+};
+
+class ZStoreBarrierBuffer : public CHeapObj<mtGC> {
+  friend class ZVerify;
+
+private:
+  static const size_t _buffer_length     = 32;
+  static const size_t _buffer_size_bytes = _buffer_length * sizeof(ZStoreBarrierEntry);
+
+  ZStoreBarrierEntry _buffer[_buffer_length];
+
+  // Color from previous phase this buffer was processed
+  uintptr_t          _last_processed_color;
+
+  // Use as a claim mechanism for installing base pointers
+  uintptr_t          _last_installed_color;
+
+  ZLock              _base_pointer_lock;
+  zaddress_unsafe    _base_pointers[_buffer_length];
+
+  // sizeof(ZStoreBarrierEntry) scaled index growing downwards
+  size_t             _current;
+
+  void on_new_phase_relocate(int i);
+  void on_new_phase_remember(int i);
+  void on_new_phase_mark(int i);
+
+  void clear();
+
+  bool is_old_mark() const;
+  bool stored_during_old_mark() const;
+  bool is_empty() const;
+  intptr_t current() const;
+
+  void install_base_pointers_inner();
+
+  void on_error(outputStream* st);
+  class OnError;
+
+public:
+  ZStoreBarrierBuffer();
+
+  static ByteSize buffer_offset();
+  static ByteSize current_offset();
+
+  static ZStoreBarrierBuffer* buffer_for_store(bool heal);
+
+  void initialize();
+  void on_new_phase();
+
+  void install_base_pointers();
+
+  void flush();
+  void add(volatile zpointer* p, zpointer prev);
+
+  // Check if p is contained in any store barrier buffer entry in the system
+  static bool is_in(volatile zpointer* p);
+};
+
+#endif // SHARE_GC_Z_ZSTOREBARRIERBUFFER_HPP
diff --git a/src/hotspot/share/gc/z/zStoreBarrierBuffer.inline.hpp b/src/hotspot/share/gc/z/zStoreBarrierBuffer.inline.hpp
new file mode 100644
index 00000000000..762aac3ccd5
--- /dev/null
+++ b/src/hotspot/share/gc/z/zStoreBarrierBuffer.inline.hpp
@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZSTOREBARRIERBUFFER_INLINE_HPP
+#define SHARE_GC_Z_ZSTOREBARRIERBUFFER_INLINE_HPP
+
+#include "gc/z/zStoreBarrierBuffer.hpp"
+
+#include "gc/shared/gc_globals.hpp"
+#include "gc/z/zThreadLocalData.hpp"
+#include "runtime/thread.hpp"
+
+inline intptr_t ZStoreBarrierBuffer::current() const {
+  return _current / sizeof(ZStoreBarrierEntry);
+}
+
+inline void ZStoreBarrierBuffer::add(volatile zpointer* p, zpointer prev) {
+  assert(ZBufferStoreBarriers, "Only buffer stores when it is enabled");
+  if (_current == 0) {
+    flush();
+  }
+  _current -= sizeof(ZStoreBarrierEntry);
+  _buffer[current()] = {p, prev};
+}
+
+inline ZStoreBarrierBuffer* ZStoreBarrierBuffer::buffer_for_store(bool heal) {
+  if (heal) {
+    return nullptr;
+  }
+
+  Thread* const thread = Thread::current();
+  if (!thread->is_Java_thread()) {
+    return nullptr;
+  }
+
+  ZStoreBarrierBuffer* const buffer = ZThreadLocalData::store_barrier_buffer(JavaThread::cast(thread));
+  return ZBufferStoreBarriers ? buffer : nullptr;
+}
+
+#endif // SHARE_GC_Z_ZSTOREBARRIERBUFFER_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zTask.cpp b/src/hotspot/share/gc/z/zTask.cpp
index 7a0503a4fb8..4e6d70c6c53 100644
--- a/src/hotspot/share/gc/z/zTask.cpp
+++ b/src/hotspot/share/gc/z/zTask.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,16 +23,13 @@
 
 #include "precompiled.hpp"
 #include "gc/z/zTask.hpp"
-#include "gc/z/zThread.hpp"
 
 ZTask::Task::Task(ZTask* task, const char* name) :
     WorkerTask(name),
     _task(task) {}
 
 void ZTask::Task::work(uint worker_id) {
-  ZThread::set_worker_id(worker_id);
   _task->work();
-  ZThread::clear_worker_id();
 }
 
 ZTask::ZTask(const char* name) :
@@ -45,3 +42,8 @@ const char* ZTask::name() const {
 WorkerTask* ZTask::worker_task() {
   return &_worker_task;
 }
+
+ZRestartableTask::ZRestartableTask(const char* name) :
+    ZTask(name) {}
+
+void ZRestartableTask::resize_workers(uint nworkers) {}
diff --git a/src/hotspot/share/gc/z/zTask.hpp b/src/hotspot/share/gc/z/zTask.hpp
index 80836aedcac..37733f8873a 100644
--- a/src/hotspot/share/gc/z/zTask.hpp
+++ b/src/hotspot/share/gc/z/zTask.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -50,4 +50,10 @@ public:
   virtual void work() = 0;
 };
 
+class ZRestartableTask : public ZTask {
+public:
+  ZRestartableTask(const char* name);
+  virtual void resize_workers(uint nworkers);
+};
+
 #endif // SHARE_GC_Z_ZTASK_HPP
diff --git a/src/hotspot/share/gc/z/zThread.cpp b/src/hotspot/share/gc/z/zThread.cpp
index c0a9d5046c4..3203da6430f 100644
--- a/src/hotspot/share/gc/z/zThread.cpp
+++ b/src/hotspot/share/gc/z/zThread.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,59 +22,29 @@
  */
 
 #include "precompiled.hpp"
-#include "gc/z/zThread.inline.hpp"
-#include "runtime/javaThread.hpp"
-#include "runtime/nonJavaThread.hpp"
-#include "utilities/debug.hpp"
+#include "gc/z/zThread.hpp"
+#include "runtime/mutexLocker.hpp"
 
-THREAD_LOCAL bool      ZThread::_initialized;
-THREAD_LOCAL uintptr_t ZThread::_id;
-THREAD_LOCAL bool      ZThread::_is_vm;
-THREAD_LOCAL bool      ZThread::_is_java;
-THREAD_LOCAL bool      ZThread::_is_worker;
-THREAD_LOCAL uint      ZThread::_worker_id;
+void ZThread::run_service() {
+  run_thread();
 
-void ZThread::initialize() {
-  assert(!_initialized, "Already initialized");
-  const Thread* const thread = Thread::current();
-  _initialized = true;
-  _id = (uintptr_t)thread;
-  _is_vm = thread->is_VM_thread();
-  _is_java = thread->is_Java_thread();
-  _is_worker = false;
-  _worker_id = (uint)-1;
+  MonitorLocker ml(Terminator_lock, Monitor::_no_safepoint_check_flag);
+
+  // Wait for signal to terminate
+  while (!should_terminate()) {
+    ml.wait();
+  }
 }
 
-const char* ZThread::name() {
-  const Thread* const thread = Thread::current();
-  if (thread->is_Named_thread()) {
-    const NamedThread* const named = (const NamedThread*)thread;
-    return named->name();
-  } else if (thread->is_Java_thread()) {
-    return "Java";
+void ZThread::stop_service() {
+  {
+    // Signal thread to terminate
+    // The should_terminate() flag should be true, and this notifies waiters
+    // to wake up.
+    MonitorLocker ml(Terminator_lock);
+    assert(should_terminate(), "This should be called when should_terminate has been set");
+    ml.notify_all();
   }
 
-  return "Unknown";
-}
-
-void ZThread::set_worker() {
-  ensure_initialized();
-  _is_worker = true;
-}
-
-bool ZThread::has_worker_id() {
-  return _initialized &&
-         _is_worker &&
-         _worker_id != (uint)-1;
-}
-
-void ZThread::set_worker_id(uint worker_id) {
-  ensure_initialized();
-  assert(!has_worker_id(), "Worker id already initialized");
-  _worker_id = worker_id;
-}
-
-void ZThread::clear_worker_id() {
-  assert(has_worker_id(), "Worker id not initialized");
-  _worker_id = (uint)-1;
+  terminate();
 }
diff --git a/src/hotspot/share/gc/z/zThread.hpp b/src/hotspot/share/gc/z/zThread.hpp
index c67807ff96e..c0b09ace465 100644
--- a/src/hotspot/share/gc/z/zThread.hpp
+++ b/src/hotspot/share/gc/z/zThread.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,38 +24,19 @@
 #ifndef SHARE_GC_Z_ZTHREAD_HPP
 #define SHARE_GC_Z_ZTHREAD_HPP
 
-#include "memory/allStatic.hpp"
-#include "utilities/globalDefinitions.hpp"
+#include "gc/shared/concurrentGCThread.hpp"
 
-class ZThread : public AllStatic {
-  friend class ZTask;
-  friend class ZWorkersInitializeTask;
-  friend class ZRuntimeWorkersInitializeTask;
+// A ZThread is a ConcurrentGCThread with some ZGC-specific handling of GC shutdown
 
+class ZThread : public ConcurrentGCThread {
 private:
-  static THREAD_LOCAL bool      _initialized;
-  static THREAD_LOCAL uintptr_t _id;
-  static THREAD_LOCAL bool      _is_vm;
-  static THREAD_LOCAL bool      _is_java;
-  static THREAD_LOCAL bool      _is_worker;
-  static THREAD_LOCAL uint      _worker_id;
-
-  static void initialize();
-  static void ensure_initialized();
-
-  static void set_worker();
-
-  static bool has_worker_id();
-  static void set_worker_id(uint worker_id);
-  static void clear_worker_id();
+  virtual void run_service();
+  virtual void stop_service();
 
 public:
-  static const char* name();
-  static uintptr_t id();
-  static bool is_vm();
-  static bool is_java();
-  static bool is_worker();
-  static uint worker_id();
+
+  virtual void run_thread() = 0;
+  virtual void terminate() = 0;
 };
 
 #endif // SHARE_GC_Z_ZTHREAD_HPP
diff --git a/src/hotspot/share/gc/z/zThreadLocalAllocBuffer.cpp b/src/hotspot/share/gc/z/zThreadLocalAllocBuffer.cpp
index fcad13ab6c0..530db62f258 100644
--- a/src/hotspot/share/gc/z/zThreadLocalAllocBuffer.cpp
+++ b/src/hotspot/share/gc/z/zThreadLocalAllocBuffer.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,11 +31,11 @@
 #include "runtime/javaThread.hpp"
 #include "runtime/stackWatermarkSet.inline.hpp"
 
-ZPerWorker<ThreadLocalAllocStats>* ZThreadLocalAllocBuffer::_stats = NULL;
+ZPerWorker<ThreadLocalAllocStats>* ZThreadLocalAllocBuffer::_stats = nullptr;
 
 void ZThreadLocalAllocBuffer::initialize() {
   if (UseTLAB) {
-    assert(_stats == NULL, "Already initialized");
+    assert(_stats == nullptr, "Already initialized");
     _stats = new ZPerWorker<ThreadLocalAllocStats>();
     reset_statistics();
   }
@@ -63,14 +63,9 @@ void ZThreadLocalAllocBuffer::publish_statistics() {
   }
 }
 
-static void fixup_address(HeapWord** p) {
-  *p = (HeapWord*)ZAddress::good_or_null((uintptr_t)*p);
-}
-
 void ZThreadLocalAllocBuffer::retire(JavaThread* thread, ThreadLocalAllocStats* stats) {
   if (UseTLAB) {
     stats->reset();
-    thread->tlab().addresses_do(fixup_address);
     thread->tlab().retire(stats);
     if (ResizeTLAB) {
       thread->tlab().resize();
@@ -78,12 +73,6 @@ void ZThreadLocalAllocBuffer::retire(JavaThread* thread, ThreadLocalAllocStats*
   }
 }
 
-void ZThreadLocalAllocBuffer::remap(JavaThread* thread) {
-  if (UseTLAB) {
-    thread->tlab().addresses_do(fixup_address);
-  }
-}
-
 void ZThreadLocalAllocBuffer::update_stats(JavaThread* thread) {
   if (UseTLAB) {
     ZStackWatermark* const watermark = StackWatermarkSet::get<ZStackWatermark>(thread, StackWatermarkKind::gc);
diff --git a/src/hotspot/share/gc/z/zThreadLocalAllocBuffer.hpp b/src/hotspot/share/gc/z/zThreadLocalAllocBuffer.hpp
index 086c8a5c351..0a4cc3b54df 100644
--- a/src/hotspot/share/gc/z/zThreadLocalAllocBuffer.hpp
+++ b/src/hotspot/share/gc/z/zThreadLocalAllocBuffer.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -41,7 +41,6 @@ public:
   static void publish_statistics();
 
   static void retire(JavaThread* thread, ThreadLocalAllocStats* stats);
-  static void remap(JavaThread* thread);
   static void update_stats(JavaThread* thread);
 };
 
diff --git a/src/hotspot/share/gc/z/zThreadLocalData.hpp b/src/hotspot/share/gc/z/zThreadLocalData.hpp
index f8c362c74ed..17f7ee9b099 100644
--- a/src/hotspot/share/gc/z/zThreadLocalData.hpp
+++ b/src/hotspot/share/gc/z/zThreadLocalData.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,22 +24,42 @@
 #ifndef SHARE_GC_Z_ZTHREADLOCALDATA_HPP
 #define SHARE_GC_Z_ZTHREADLOCALDATA_HPP
 
+#include "gc/z/zAddress.hpp"
+#include "gc/z/zGenerationId.hpp"
 #include "gc/z/zMarkStack.hpp"
-#include "gc/z/zGlobals.hpp"
+#include "gc/z/zStoreBarrierBuffer.hpp"
 #include "runtime/javaThread.hpp"
 #include "utilities/debug.hpp"
 #include "utilities/sizes.hpp"
 
 class ZThreadLocalData {
 private:
-  uintptr_t              _address_bad_mask;
-  ZMarkThreadLocalStacks _stacks;
-  oop*                   _invisible_root;
+  uintptr_t              _load_good_mask;
+  uintptr_t              _load_bad_mask;
+  uintptr_t              _mark_bad_mask;
+  uintptr_t              _store_good_mask;
+  uintptr_t              _store_bad_mask;
+  uintptr_t              _uncolor_mask;
+  uintptr_t              _nmethod_disarmed;
+  ZStoreBarrierBuffer*   _store_barrier_buffer;
+  ZMarkThreadLocalStacks _mark_stacks[2];
+  zaddress_unsafe*       _invisible_root;
 
   ZThreadLocalData() :
-      _address_bad_mask(0),
-      _stacks(),
-      _invisible_root(NULL) {}
+      _load_good_mask(0),
+      _load_bad_mask(0),
+      _mark_bad_mask(0),
+      _store_good_mask(0),
+      _store_bad_mask(0),
+      _uncolor_mask(0),
+      _nmethod_disarmed(0),
+      _store_barrier_buffer(new ZStoreBarrierBuffer()),
+      _mark_stacks(),
+      _invisible_root(nullptr) {}
+
+  ~ZThreadLocalData() {
+    delete _store_barrier_buffer;
+  }
 
   static ZThreadLocalData* data(Thread* thread) {
     return thread->gc_data<ZThreadLocalData>();
@@ -54,37 +74,74 @@ public:
     data(thread)->~ZThreadLocalData();
   }
 
-  static void set_address_bad_mask(Thread* thread, uintptr_t mask) {
-    data(thread)->_address_bad_mask = mask;
+  static void set_load_bad_mask(Thread* thread, uintptr_t mask) {
+    data(thread)->_load_bad_mask = mask;
   }
 
-  static ZMarkThreadLocalStacks* stacks(Thread* thread) {
-    return &data(thread)->_stacks;
+  static void set_mark_bad_mask(Thread* thread, uintptr_t mask) {
+    data(thread)->_mark_bad_mask = mask;
   }
 
-  static void set_invisible_root(Thread* thread, oop* root) {
-    assert(data(thread)->_invisible_root == NULL, "Already set");
+  static void set_store_bad_mask(Thread* thread, uintptr_t mask) {
+    data(thread)->_store_bad_mask = mask;
+  }
+
+  static void set_load_good_mask(Thread* thread, uintptr_t mask) {
+    data(thread)->_load_good_mask = mask;
+  }
+
+  static void set_store_good_mask(Thread* thread, uintptr_t mask) {
+    data(thread)->_store_good_mask = mask;
+  }
+
+  static void set_nmethod_disarmed(Thread* thread, uintptr_t value) {
+    data(thread)->_nmethod_disarmed = value;
+  }
+
+  static ZMarkThreadLocalStacks* mark_stacks(Thread* thread, ZGenerationId id) {
+    return &data(thread)->_mark_stacks[(int)id];
+  }
+
+  static ZStoreBarrierBuffer* store_barrier_buffer(Thread* thread) {
+    return data(thread)->_store_barrier_buffer;
+  }
+
+  static void set_invisible_root(Thread* thread, zaddress_unsafe* root) {
+    assert(data(thread)->_invisible_root == nullptr, "Already set");
     data(thread)->_invisible_root = root;
   }
 
   static void clear_invisible_root(Thread* thread) {
-    assert(data(thread)->_invisible_root != NULL, "Should be set");
-    data(thread)->_invisible_root = NULL;
+    assert(data(thread)->_invisible_root != nullptr, "Should be set");
+    data(thread)->_invisible_root = nullptr;
   }
 
-  template <typename T>
-  static void do_invisible_root(Thread* thread, T f) {
-    if (data(thread)->_invisible_root != NULL) {
-      f(data(thread)->_invisible_root);
-    }
+  static zaddress_unsafe* invisible_root(Thread* thread) {
+    return data(thread)->_invisible_root;
   }
 
-  static ByteSize address_bad_mask_offset() {
-    return Thread::gc_data_offset() + byte_offset_of(ZThreadLocalData, _address_bad_mask);
+  static ByteSize load_bad_mask_offset() {
+    return Thread::gc_data_offset() + byte_offset_of(ZThreadLocalData, _load_bad_mask);
+  }
+
+  static ByteSize mark_bad_mask_offset() {
+    return Thread::gc_data_offset() + byte_offset_of(ZThreadLocalData, _mark_bad_mask);
+  }
+
+  static ByteSize store_bad_mask_offset() {
+    return Thread::gc_data_offset() + byte_offset_of(ZThreadLocalData, _store_bad_mask);
+  }
+
+  static ByteSize store_good_mask_offset() {
+    return Thread::gc_data_offset() + byte_offset_of(ZThreadLocalData, _store_good_mask);
   }
 
   static ByteSize nmethod_disarmed_offset() {
-    return address_bad_mask_offset() + in_ByteSize(ZAddressBadMaskHighOrderBitsOffset);
+    return Thread::gc_data_offset() + byte_offset_of(ZThreadLocalData, _nmethod_disarmed);
+  }
+
+  static ByteSize store_barrier_buffer_offset() {
+    return Thread::gc_data_offset() + byte_offset_of(ZThreadLocalData, _store_barrier_buffer);
   }
 };
 
diff --git a/src/hotspot/share/gc/z/zTracer.cpp b/src/hotspot/share/gc/z/zTracer.cpp
index 55934629211..ddb5ca96e80 100644
--- a/src/hotspot/share/gc/z/zTracer.cpp
+++ b/src/hotspot/share/gc/z/zTracer.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
 #include "precompiled.hpp"
 #include "gc/shared/gcId.hpp"
 #include "gc/z/zGlobals.hpp"
+#include "gc/z/zPageType.hpp"
 #include "gc/z/zStat.hpp"
 #include "gc/z/zTracer.hpp"
 #include "jfr/jfrEvents.hpp"
@@ -40,11 +41,11 @@ class ZPageTypeConstant : public JfrSerializer {
 public:
   virtual void serialize(JfrCheckpointWriter& writer) {
     writer.write_count(3);
-    writer.write_key(ZPageTypeSmall);
+    writer.write_key((u8)ZPageType::small);
     writer.write("Small");
-    writer.write_key(ZPageTypeMedium);
+    writer.write_key((u8)ZPageType::medium);
     writer.write("Medium");
-    writer.write_key(ZPageTypeLarge);
+    writer.write_key((u8)ZPageType::large);
     writer.write("Large");
   }
 };
@@ -53,7 +54,7 @@ class ZStatisticsCounterTypeConstant : public JfrSerializer {
 public:
   virtual void serialize(JfrCheckpointWriter& writer) {
     writer.write_count(ZStatCounter::count());
-    for (ZStatCounter* counter = ZStatCounter::first(); counter != NULL; counter = counter->next()) {
+    for (ZStatCounter* counter = ZStatCounter::first(); counter != nullptr; counter = counter->next()) {
       writer.write_key(counter->id());
       writer.write(counter->name());
     }
@@ -64,7 +65,7 @@ class ZStatisticsSamplerTypeConstant : public JfrSerializer {
 public:
   virtual void serialize(JfrCheckpointWriter& writer) {
     writer.write_count(ZStatSampler::count());
-    for (ZStatSampler* sampler = ZStatSampler::first(); sampler != NULL; sampler = sampler->next()) {
+    for (ZStatSampler* sampler = ZStatSampler::first(); sampler != nullptr; sampler = sampler->next()) {
       writer.write_key(sampler->id());
       writer.write(sampler->name());
     }
@@ -85,14 +86,39 @@ static void register_jfr_type_serializers() {
 
 #endif // INCLUDE_JFR
 
-ZTracer* ZTracer::_tracer = NULL;
+ZMinorTracer::ZMinorTracer() :
+    GCTracer(ZMinor) {
+}
 
-ZTracer::ZTracer() :
-    GCTracer(Z) {}
+ZMajorTracer::ZMajorTracer() :
+    GCTracer(ZMajor) {}
+
+void ZGenerationTracer::report_start(const Ticks& timestamp) {
+  _start = timestamp;
+}
+
+void ZYoungTracer::report_end(const Ticks& timestamp) {
+  NoSafepointVerifier nsv;
+
+  EventZYoungGarbageCollection e(UNTIMED);
+  e.set_gcId(GCId::current());
+  e.set_tenuringThreshold(ZGeneration::young()->tenuring_threshold());
+  e.set_starttime(_start);
+  e.set_endtime(timestamp);
+  e.commit();
+}
+
+void ZOldTracer::report_end(const Ticks& timestamp) {
+  NoSafepointVerifier nsv;
+
+  EventZOldGarbageCollection e(UNTIMED);
+  e.set_gcId(GCId::current());
+  e.set_starttime(_start);
+  e.set_endtime(timestamp);
+  e.commit();
+}
 
 void ZTracer::initialize() {
-  assert(_tracer == NULL, "Already initialized");
-  _tracer = new ZTracer();
   JFR_ONLY(register_jfr_type_serializers());
 }
 
diff --git a/src/hotspot/share/gc/z/zTracer.hpp b/src/hotspot/share/gc/z/zTracer.hpp
index fbf1a9346fc..70325732416 100644
--- a/src/hotspot/share/gc/z/zTracer.hpp
+++ b/src/hotspot/share/gc/z/zTracer.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,30 +25,58 @@
 #define SHARE_GC_Z_ZTRACER_HPP
 
 #include "gc/shared/gcTrace.hpp"
+#include "gc/z/zGenerationId.hpp"
 
 class ZStatCounter;
 class ZStatPhase;
 class ZStatSampler;
 
-class ZTracer : public GCTracer, public CHeapObj<mtGC> {
+class ZTracer : AllStatic {
 private:
-  static ZTracer* _tracer;
-
-  ZTracer();
-
-  void send_stat_counter(const ZStatCounter& counter, uint64_t increment, uint64_t value);
-  void send_stat_sampler(const ZStatSampler& sampler, uint64_t value);
-  void send_thread_phase(const char* name, const Ticks& start, const Ticks& end);
-  void send_thread_debug(const char* name, const Ticks& start, const Ticks& end);
+  static void send_stat_counter(const ZStatCounter& counter, uint64_t increment, uint64_t value);
+  static void send_stat_sampler(const ZStatSampler& sampler, uint64_t value);
+  static void send_thread_phase(const char* name, const Ticks& start, const Ticks& end);
+  static void send_thread_debug(const char* name, const Ticks& start, const Ticks& end);
 
 public:
-  static ZTracer* tracer();
   static void initialize();
 
-  void report_stat_counter(const ZStatCounter& counter, uint64_t increment, uint64_t value);
-  void report_stat_sampler(const ZStatSampler& sampler, uint64_t value);
-  void report_thread_phase(const char* name, const Ticks& start, const Ticks& end);
-  void report_thread_debug(const char* name, const Ticks& start, const Ticks& end);
+  static void report_stat_counter(const ZStatCounter& counter, uint64_t increment, uint64_t value);
+  static void report_stat_sampler(const ZStatSampler& sampler, uint64_t value);
+  static void report_thread_phase(const char* name, const Ticks& start, const Ticks& end);
+  static void report_thread_debug(const char* name, const Ticks& start, const Ticks& end);
+};
+
+class ZMinorTracer : public GCTracer {
+public:
+  ZMinorTracer();
+};
+
+class ZMajorTracer : public GCTracer {
+public:
+  ZMajorTracer();
+};
+
+class ZGenerationTracer {
+protected:
+  Ticks _start;
+
+public:
+  ZGenerationTracer() :
+      _start() {}
+
+  void report_start(const Ticks& timestamp);
+  virtual void report_end(const Ticks& timestamp) = 0;
+};
+
+class ZYoungTracer : public ZGenerationTracer {
+public:
+  void report_end(const Ticks& timestamp) override;
+};
+
+class ZOldTracer : public ZGenerationTracer {
+public:
+  void report_end(const Ticks& timestamp) override;
 };
 
 // For temporary latency measurements during development and debugging
diff --git a/src/hotspot/share/gc/z/zTracer.inline.hpp b/src/hotspot/share/gc/z/zTracer.inline.hpp
index cfaf7b43f0c..83a0fe47194 100644
--- a/src/hotspot/share/gc/z/zTracer.inline.hpp
+++ b/src/hotspot/share/gc/z/zTracer.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,10 +28,6 @@
 
 #include "jfr/jfrEvents.hpp"
 
-inline ZTracer* ZTracer::tracer() {
-  return _tracer;
-}
-
 inline void ZTracer::report_stat_counter(const ZStatCounter& counter, uint64_t increment, uint64_t value) {
   if (EventZStatisticsCounter::is_enabled()) {
     send_stat_counter(counter, increment, value);
@@ -61,7 +57,7 @@ inline ZTraceThreadDebug::ZTraceThreadDebug(const char* name) :
     _name(name) {}
 
 inline ZTraceThreadDebug::~ZTraceThreadDebug() {
-  ZTracer::tracer()->report_thread_debug(_name, _start, Ticks::now());
+  ZTracer::report_thread_debug(_name, _start, Ticks::now());
 }
 
 #endif // SHARE_GC_Z_ZTRACER_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zUncoloredRoot.cpp b/src/hotspot/share/gc/z/zUncoloredRoot.cpp
new file mode 100644
index 00000000000..505e10628d7
--- /dev/null
+++ b/src/hotspot/share/gc/z/zUncoloredRoot.cpp
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zUncoloredRoot.inline.hpp"
+
+void ZUncoloredRootClosure::do_oop(oop* p) {
+  do_root(ZUncoloredRoot::cast(p));
+}
+
+void ZUncoloredRootClosure::do_oop(narrowOop* p) {
+  ShouldNotReachHere();
+}
diff --git a/src/hotspot/share/gc/z/zUncoloredRoot.hpp b/src/hotspot/share/gc/z/zUncoloredRoot.hpp
new file mode 100644
index 00000000000..e980bd70eb2
--- /dev/null
+++ b/src/hotspot/share/gc/z/zUncoloredRoot.hpp
@@ -0,0 +1,145 @@
+/*
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZUNCOLOREDROOT_HPP
+#define SHARE_GC_Z_ZUNCOLOREDROOT_HPP
+
+#include "gc/z/zAddress.hpp"
+#include "memory/allStatic.hpp"
+#include "memory/iterator.hpp"
+#include "oops/oopsHierarchy.hpp"
+
+// ZGC has two types of oops:
+//
+// Colored oops (zpointer)
+//   Metadata explicitly encoded in the pointer bits.
+//   Requires normal GC barriers to use.
+//   - OopStorage oops.
+//
+// Uncolored oops (zaddress, zaddress_unsafe)
+//   Metadata is either implicit or stored elsewhere
+//   Requires specialized GC barriers
+//   - nmethod oops - nmethod entry barriers
+//   - Thread oops - stack watermark barriers
+//
+// Even though the uncolored roots lack the color/metadata, ZGC still needs
+// that information when processing the roots. Therefore, we store the color
+// in the "container" object where the oop is located, and use specialized
+// GC barriers, which accepts the external color as an extra argument. These
+// roots are handled in this file.
+//
+// The zaddress_unsafe type is used to hold uncolored oops that the GC needs
+// to process before it is safe to use. E.g. the original object might have
+// been relocated and the address needs to be updated. The zaddress type
+// denotes that this pointer refers the the correct address of the object.
+
+class ZUncoloredRoot : public AllStatic {
+private:
+  template <typename ObjectFunctionT>
+  static void barrier(ObjectFunctionT function, zaddress_unsafe* p, uintptr_t color);
+
+  static zaddress make_load_good(zaddress_unsafe addr, uintptr_t color);
+
+public:
+  // Operations to be used on oops that are known to be load good
+  static void mark_object(zaddress addr);
+  static void mark_invisible_object(zaddress addr);
+  static void keep_alive_object(zaddress addr);
+  static void mark_young_object(zaddress addr);
+
+  // Operations on roots, with an externally provided color
+  static void mark(zaddress_unsafe* p, uintptr_t color);
+  static void mark_young(zaddress_unsafe* p, uintptr_t color);
+  static void process(zaddress_unsafe* p, uintptr_t color);
+  static void process_invisible(zaddress_unsafe* p, uintptr_t color);
+  static void process_weak(zaddress_unsafe* p, uintptr_t color);
+  static void process_no_keepalive(zaddress_unsafe* p, uintptr_t color);
+
+  // Cast needed when ZGC interfaces with the rest of the JVM,
+  // which is agnostic to ZGC's oop type system.
+  static zaddress_unsafe* cast(oop* p);
+
+  typedef void (*RootFunction)(zaddress_unsafe*, uintptr_t);
+  typedef void (*ObjectFunction)(zaddress);
+};
+
+class ZUncoloredRootClosure : public OopClosure {
+private:
+  void do_oop(oop* p) final;
+  void do_oop(narrowOop* p) final;
+
+public:
+  virtual void do_root(zaddress_unsafe* p) = 0;
+};
+
+class ZUncoloredRootMarkOopClosure : public ZUncoloredRootClosure {
+private:
+  const uintptr_t _color;
+
+public:
+  ZUncoloredRootMarkOopClosure(uintptr_t color);
+
+  virtual void do_root(zaddress_unsafe* p);
+};
+
+class ZUncoloredRootMarkYoungOopClosure : public ZUncoloredRootClosure {
+private:
+  const uintptr_t _color;
+
+public:
+  ZUncoloredRootMarkYoungOopClosure(uintptr_t color);
+
+  virtual void do_root(zaddress_unsafe* p);
+};
+
+class ZUncoloredRootProcessOopClosure : public ZUncoloredRootClosure {
+private:
+  const uintptr_t _color;
+
+public:
+  ZUncoloredRootProcessOopClosure(uintptr_t color);
+
+  virtual void do_root(zaddress_unsafe* p);
+};
+
+class ZUncoloredRootProcessWeakOopClosure : public ZUncoloredRootClosure {
+private:
+  const uintptr_t _color;
+
+public:
+  ZUncoloredRootProcessWeakOopClosure(uintptr_t color);
+
+  virtual void do_root(zaddress_unsafe* p);
+};
+
+class ZUncoloredRootProcessNoKeepaliveOopClosure : public ZUncoloredRootClosure {
+private:
+  const uintptr_t _color;
+
+public:
+  ZUncoloredRootProcessNoKeepaliveOopClosure(uintptr_t color);
+
+  virtual void do_root(zaddress_unsafe* p);
+};
+
+#endif // SHARE_GC_Z_ZUNCOLOREDROOT_HPP
diff --git a/src/hotspot/share/gc/z/zUncoloredRoot.inline.hpp b/src/hotspot/share/gc/z/zUncoloredRoot.inline.hpp
new file mode 100644
index 00000000000..a9885f94f19
--- /dev/null
+++ b/src/hotspot/share/gc/z/zUncoloredRoot.inline.hpp
@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZUNCOLOREDROOT_INLINE_HPP
+#define SHARE_GC_Z_ZUNCOLOREDROOT_INLINE_HPP
+
+#include "gc/z/zUncoloredRoot.hpp"
+
+#include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zBarrier.inline.hpp"
+#include "gc/z/zHeap.inline.hpp"
+#include "oops/oop.hpp"
+
+template <typename ObjectFunctionT>
+inline void ZUncoloredRoot::barrier(ObjectFunctionT function, zaddress_unsafe* p, uintptr_t color) {
+  z_assert_is_barrier_safe();
+
+  const zaddress_unsafe addr = Atomic::load(p);
+  assert_is_valid(addr);
+
+  // Nothing to do for nulls
+  if (is_null(addr)) {
+    return;
+  }
+
+  // Make load good
+  const zaddress load_good_addr = make_load_good(addr, color);
+
+  // Apply function
+  function(load_good_addr);
+
+  // Non-atomic healing helps speed up root scanning. This is safe to do
+  // since we are always healing roots in a safepoint, or under a lock,
+  // which ensures we are never racing with mutators modifying roots while
+  // we are healing them. It's also safe in case multiple GC threads try
+  // to heal the same root if it is aligned, since they would always heal
+  // the root in the same way and it does not matter in which order it
+  // happens. For misaligned oops, there needs to be mutual exclusion.
+  *(zaddress*)p = load_good_addr;
+}
+
+inline zaddress ZUncoloredRoot::make_load_good(zaddress_unsafe addr, uintptr_t color) {
+  const zpointer color_ptr = ZAddress::color(zaddress::null, color);
+  if (!ZPointer::is_load_good(color_ptr)) {
+    return ZBarrier::relocate_or_remap(addr, ZBarrier::remap_generation(color_ptr));
+  } else {
+    return safe(addr);
+  }
+}
+
+inline void ZUncoloredRoot::mark_object(zaddress addr) {
+  ZBarrier::mark<ZMark::DontResurrect, ZMark::AnyThread, ZMark::Follow, ZMark::Strong>(addr);
+}
+
+inline void ZUncoloredRoot::mark_young_object(zaddress addr) {
+  ZBarrier::mark_if_young<ZMark::DontResurrect, ZMark::GCThread, ZMark::Follow>(addr);
+}
+
+inline void ZUncoloredRoot::mark_invisible_object(zaddress addr) {
+  ZBarrier::mark<ZMark::DontResurrect, ZMark::AnyThread, ZMark::DontFollow, ZMark::Strong>(addr);
+}
+
+inline void ZUncoloredRoot::keep_alive_object(zaddress addr) {
+  ZBarrier::mark<ZMark::Resurrect, ZMark::AnyThread, ZMark::Follow, ZMark::Strong>(addr);
+}
+
+inline void ZUncoloredRoot::mark(zaddress_unsafe* p, uintptr_t color) {
+  barrier(mark_object, p, color);
+}
+
+inline void ZUncoloredRoot::mark_young(zaddress_unsafe* p, uintptr_t color) {
+  barrier(mark_young_object, p, color);
+}
+
+inline void ZUncoloredRoot::process(zaddress_unsafe* p, uintptr_t color) {
+  barrier(mark_object, p, color);
+}
+
+inline void ZUncoloredRoot::process_invisible(zaddress_unsafe* p, uintptr_t color) {
+  barrier(mark_invisible_object, p, color);
+}
+
+inline void ZUncoloredRoot::process_weak(zaddress_unsafe* p, uintptr_t color) {
+  barrier(keep_alive_object, p, color);
+}
+
+inline void ZUncoloredRoot::process_no_keepalive(zaddress_unsafe* p, uintptr_t color) {
+  auto do_nothing = [](zaddress) -> void {};
+  barrier(do_nothing, p, color);
+}
+
+inline zaddress_unsafe* ZUncoloredRoot::cast(oop* p) {
+  zaddress_unsafe* const root = (zaddress_unsafe*)p;
+  DEBUG_ONLY(assert_is_valid(*root);)
+  return root;
+}
+
+inline ZUncoloredRootMarkOopClosure::ZUncoloredRootMarkOopClosure(uintptr_t color) :
+    _color(color) {}
+
+inline void ZUncoloredRootMarkOopClosure::do_root(zaddress_unsafe* p) {
+  ZUncoloredRoot::mark(p, _color);
+}
+
+inline ZUncoloredRootMarkYoungOopClosure::ZUncoloredRootMarkYoungOopClosure(uintptr_t color) :
+    _color(color) {}
+
+inline void ZUncoloredRootMarkYoungOopClosure::do_root(zaddress_unsafe* p) {
+  ZUncoloredRoot::mark_young(p, _color);
+}
+
+inline ZUncoloredRootProcessOopClosure::ZUncoloredRootProcessOopClosure(uintptr_t color) :
+    _color(color) {}
+
+inline void ZUncoloredRootProcessOopClosure::do_root(zaddress_unsafe* p) {
+  ZUncoloredRoot::process(p, _color);
+}
+
+inline ZUncoloredRootProcessWeakOopClosure::ZUncoloredRootProcessWeakOopClosure(uintptr_t color) :
+  _color(color) {}
+
+inline void ZUncoloredRootProcessWeakOopClosure::do_root(zaddress_unsafe* p) {
+  ZUncoloredRoot::process_weak(p, _color);
+}
+
+inline ZUncoloredRootProcessNoKeepaliveOopClosure::ZUncoloredRootProcessNoKeepaliveOopClosure(uintptr_t color) :
+    _color(color) {}
+
+inline void ZUncoloredRootProcessNoKeepaliveOopClosure::do_root(zaddress_unsafe* p) {
+  ZUncoloredRoot::process_no_keepalive(p, _color);
+}
+
+#endif // SHARE_GC_Z_ZUNCOLOREDROOT_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/zUncommitter.cpp b/src/hotspot/share/gc/z/zUncommitter.cpp
index 27ca4f4c193..fb8cf29194f 100644
--- a/src/hotspot/share/gc/z/zUncommitter.cpp
+++ b/src/hotspot/share/gc/z/zUncommitter.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -59,7 +59,7 @@ bool ZUncommitter::should_continue() const {
   return !_stop;
 }
 
-void ZUncommitter::run_service() {
+void ZUncommitter::run_thread() {
   uint64_t timeout = 0;
 
   while (wait(timeout)) {
@@ -89,7 +89,7 @@ void ZUncommitter::run_service() {
   }
 }
 
-void ZUncommitter::stop_service() {
+void ZUncommitter::terminate() {
   ZLocker<ZConditionLock> locker(&_lock);
   _stop = true;
   _lock.notify_all();
diff --git a/src/hotspot/share/gc/z/zUncommitter.hpp b/src/hotspot/share/gc/z/zUncommitter.hpp
index 6cb38d9db4c..b626df8dddf 100644
--- a/src/hotspot/share/gc/z/zUncommitter.hpp
+++ b/src/hotspot/share/gc/z/zUncommitter.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,12 +24,12 @@
 #ifndef SHARE_GC_Z_ZUNCOMMITTER_HPP
 #define SHARE_GC_Z_ZUNCOMMITTER_HPP
 
-#include "gc/shared/concurrentGCThread.hpp"
 #include "gc/z/zLock.hpp"
+#include "gc/z/zThread.hpp"
 
-class ZPageAllocation;
+class ZPageAllocator;
 
-class ZUncommitter : public ConcurrentGCThread {
+class ZUncommitter : public ZThread {
 private:
   ZPageAllocator* const  _page_allocator;
   mutable ZConditionLock _lock;
@@ -39,8 +39,8 @@ private:
   bool should_continue() const;
 
 protected:
-  virtual void run_service();
-  virtual void stop_service();
+  virtual void run_thread();
+  virtual void terminate();
 
 public:
   ZUncommitter(ZPageAllocator* page_allocator);
diff --git a/src/hotspot/share/gc/z/zUnload.cpp b/src/hotspot/share/gc/z/zUnload.cpp
index 0378466324e..d601916ed6d 100644
--- a/src/hotspot/share/gc/z/zUnload.cpp
+++ b/src/hotspot/share/gc/z/zUnload.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,36 +30,35 @@
 #include "gc/shared/gcBehaviours.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "gc/z/zBarrier.inline.hpp"
+#include "gc/z/zBarrierSetNMethod.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zLock.inline.hpp"
 #include "gc/z/zNMethod.hpp"
 #include "gc/z/zStat.hpp"
+#include "gc/z/zUncoloredRoot.inline.hpp"
 #include "gc/z/zUnload.hpp"
 #include "memory/metaspaceUtils.hpp"
 #include "oops/access.inline.hpp"
 
-static const ZStatSubPhase ZSubPhaseConcurrentClassesUnlink("Concurrent Classes Unlink");
-static const ZStatSubPhase ZSubPhaseConcurrentClassesPurge("Concurrent Classes Purge");
-
-class ZPhantomIsAliveObjectClosure : public BoolObjectClosure {
-public:
-  virtual bool do_object_b(oop o) {
-    return ZBarrier::is_alive_barrier_on_phantom_oop(o);
-  }
-};
+static const ZStatSubPhase ZSubPhaseConcurrentClassesUnlink("Concurrent Classes Unlink", ZGenerationId::old);
+static const ZStatSubPhase ZSubPhaseConcurrentClassesPurge("Concurrent Classes Purge", ZGenerationId::old);
 
 class ZIsUnloadingOopClosure : public OopClosure {
 private:
-  ZPhantomIsAliveObjectClosure _is_alive;
-  bool                         _is_unloading;
+  const uintptr_t _color;
+  bool            _is_unloading;
 
 public:
-  ZIsUnloadingOopClosure() :
-      _is_alive(),
+  ZIsUnloadingOopClosure(nmethod* nm) :
+      _color(ZNMethod::color(nm)),
       _is_unloading(false) {}
 
   virtual void do_oop(oop* p) {
-    const oop o = RawAccess<>::oop_load(p);
-    if (o != NULL && !_is_alive.do_object_b(o)) {
+    // Create local, aligned root
+    zaddress_unsafe addr = Atomic::load(ZUncoloredRoot::cast(p));
+    ZUncoloredRoot::process_no_keepalive(&addr, _color);
+
+    if (!is_null(addr) && ZHeap::heap()->is_old(safe(addr)) && !ZHeap::heap()->is_object_live(safe(addr))) {
       _is_unloading = true;
     }
   }
@@ -79,7 +78,11 @@ public:
     nmethod* const nm = method->as_nmethod();
     ZReentrantLock* const lock = ZNMethod::lock_for_nmethod(nm);
     ZLocker<ZReentrantLock> locker(lock);
-    ZIsUnloadingOopClosure cl;
+    if (!ZNMethod::is_armed(nm)) {
+      // Disarmed nmethods are alive
+      return false;
+    }
+    ZIsUnloadingOopClosure cl(nm);
     ZNMethod::nmethod_oops_do_inner(nm, &cl);
     return cl.is_unloading();
   }
@@ -139,13 +142,13 @@ void ZUnload::unlink() {
     return;
   }
 
-  ZStatTimer timer(ZSubPhaseConcurrentClassesUnlink);
-  SuspendibleThreadSetJoiner sts;
+  ZStatTimerOld timer(ZSubPhaseConcurrentClassesUnlink);
+  SuspendibleThreadSetJoiner sts_joiner;
   bool unloading_occurred;
 
   {
     MutexLocker ml(ClassLoaderDataGraph_lock);
-    unloading_occurred = SystemDictionary::do_unloading(ZStatPhase::timer());
+    unloading_occurred = SystemDictionary::do_unloading(ZGeneration::old()->gc_timer());
   }
 
   Klass::clean_weak_klass_links(unloading_occurred);
@@ -158,10 +161,10 @@ void ZUnload::purge() {
     return;
   }
 
-  ZStatTimer timer(ZSubPhaseConcurrentClassesPurge);
+  ZStatTimerOld timer(ZSubPhaseConcurrentClassesPurge);
 
   {
-    SuspendibleThreadSetJoiner sts;
+    SuspendibleThreadSetJoiner sts_joiner;
     ZNMethod::purge();
   }
 
diff --git a/src/hotspot/share/gc/z/zUnmapper.cpp b/src/hotspot/share/gc/z/zUnmapper.cpp
index 1997449fd5d..95ea87f01f9 100644
--- a/src/hotspot/share/gc/z/zUnmapper.cpp
+++ b/src/hotspot/share/gc/z/zUnmapper.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -45,11 +45,11 @@ ZPage* ZUnmapper::dequeue() {
 
   for (;;) {
     if (_stop) {
-      return NULL;
+      return nullptr;
     }
 
     ZPage* const page = _queue.remove_first();
-    if (page != NULL) {
+    if (page != nullptr) {
       return page;
     }
 
@@ -70,22 +70,16 @@ void ZUnmapper::do_unmap_and_destroy_page(ZPage* page) const {
 }
 
 void ZUnmapper::unmap_and_destroy_page(ZPage* page) {
-  // Asynchronous unmap and destroy is not supported with ZVerifyViews
-  if (ZVerifyViews) {
-    // Immediately unmap and destroy
-    do_unmap_and_destroy_page(page);
-  } else {
-    // Enqueue for asynchronous unmap and destroy
-    ZLocker<ZConditionLock> locker(&_lock);
-    _queue.insert_last(page);
-    _lock.notify_all();
-  }
+  // Enqueue for asynchronous unmap and destroy
+  ZLocker<ZConditionLock> locker(&_lock);
+  _queue.insert_last(page);
+  _lock.notify_all();
 }
 
-void ZUnmapper::run_service() {
+void ZUnmapper::run_thread() {
   for (;;) {
     ZPage* const page = dequeue();
-    if (page == NULL) {
+    if (page == nullptr) {
       // Stop
       return;
     }
@@ -94,7 +88,7 @@ void ZUnmapper::run_service() {
   }
 }
 
-void ZUnmapper::stop_service() {
+void ZUnmapper::terminate() {
   ZLocker<ZConditionLock> locker(&_lock);
   _stop = true;
   _lock.notify_all();
diff --git a/src/hotspot/share/gc/z/zUnmapper.hpp b/src/hotspot/share/gc/z/zUnmapper.hpp
index fd263948759..23d384e7970 100644
--- a/src/hotspot/share/gc/z/zUnmapper.hpp
+++ b/src/hotspot/share/gc/z/zUnmapper.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,14 +24,14 @@
 #ifndef SHARE_GC_Z_ZUNMAPPER_HPP
 #define SHARE_GC_Z_ZUNMAPPER_HPP
 
-#include "gc/shared/concurrentGCThread.hpp"
 #include "gc/z/zList.hpp"
 #include "gc/z/zLock.hpp"
+#include "gc/z/zThread.hpp"
 
 class ZPage;
 class ZPageAllocator;
 
-class ZUnmapper : public ConcurrentGCThread {
+class ZUnmapper : public ZThread {
 private:
   ZPageAllocator* const _page_allocator;
   ZConditionLock        _lock;
@@ -42,8 +42,8 @@ private:
   void do_unmap_and_destroy_page(ZPage* page) const;
 
 protected:
-  virtual void run_service();
-  virtual void stop_service();
+  virtual void run_thread();
+  virtual void terminate();
 
 public:
   ZUnmapper(ZPageAllocator* page_allocator);
diff --git a/src/hotspot/share/gc/z/zUtils.cpp b/src/hotspot/share/gc/z/zUtils.cpp
new file mode 100644
index 00000000000..3804baad595
--- /dev/null
+++ b/src/hotspot/share/gc/z/zUtils.cpp
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zUtils.hpp"
+#include "runtime/nonJavaThread.hpp"
+
+#include <algorithm>
+
+const char* ZUtils::thread_name() {
+  const Thread* const thread = Thread::current();
+  if (thread->is_Named_thread()) {
+    const NamedThread* const named = (const NamedThread*)thread;
+    return named->name();
+  }
+
+  return thread->type_name();
+}
+
+void ZUtils::fill(uintptr_t* addr, size_t count, uintptr_t value) {
+  std::fill_n(addr, count, value);
+}
diff --git a/src/hotspot/share/gc/z/zUtils.hpp b/src/hotspot/share/gc/z/zUtils.hpp
index 470329daf0d..f82ef06235c 100644
--- a/src/hotspot/share/gc/z/zUtils.hpp
+++ b/src/hotspot/share/gc/z/zUtils.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,11 +24,15 @@
 #ifndef SHARE_GC_Z_ZUTILS_HPP
 #define SHARE_GC_Z_ZUTILS_HPP
 
+#include "gc/z/zAddress.hpp"
 #include "memory/allStatic.hpp"
 #include "utilities/globalDefinitions.hpp"
 
 class ZUtils : public AllStatic {
 public:
+  // Thread
+  static const char* thread_name();
+
   // Allocation
   static uintptr_t alloc_aligned(size_t alignment, size_t size);
 
@@ -37,9 +41,12 @@ public:
   static size_t words_to_bytes(size_t size_in_words);
 
   // Object
-  static size_t object_size(uintptr_t addr);
-  static void object_copy_disjoint(uintptr_t from, uintptr_t to, size_t size);
-  static void object_copy_conjoint(uintptr_t from, uintptr_t to, size_t size);
+  static size_t object_size(zaddress addr);
+  static void object_copy_disjoint(zaddress from, zaddress to, size_t size);
+  static void object_copy_conjoint(zaddress from, zaddress to, size_t size);
+
+  // Memory
+  static void fill(uintptr_t* addr, size_t count, uintptr_t value);
 };
 
 #endif // SHARE_GC_Z_ZUTILS_HPP
diff --git a/src/hotspot/share/gc/z/zUtils.inline.hpp b/src/hotspot/share/gc/z/zUtils.inline.hpp
index 17fc8a69bc7..eda9fca9398 100644
--- a/src/hotspot/share/gc/z/zUtils.inline.hpp
+++ b/src/hotspot/share/gc/z/zUtils.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,7 @@
 
 #include "gc/z/zUtils.hpp"
 
-#include "gc/z/zOop.inline.hpp"
+#include "gc/z/zAddress.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "utilities/align.hpp"
 #include "utilities/copy.hpp"
@@ -42,17 +42,17 @@ inline size_t ZUtils::words_to_bytes(size_t size_in_words) {
   return size_in_words << LogBytesPerWord;
 }
 
-inline size_t ZUtils::object_size(uintptr_t addr) {
-  return words_to_bytes(ZOop::from_address(addr)->size());
+inline size_t ZUtils::object_size(zaddress addr) {
+  return words_to_bytes(to_oop(addr)->size());
 }
 
-inline void ZUtils::object_copy_disjoint(uintptr_t from, uintptr_t to, size_t size) {
-  Copy::aligned_disjoint_words((HeapWord*)from, (HeapWord*)to, bytes_to_words(size));
+inline void ZUtils::object_copy_disjoint(zaddress from, zaddress to, size_t size) {
+  Copy::aligned_disjoint_words((HeapWord*)untype(from), (HeapWord*)untype(to), bytes_to_words(size));
 }
 
-inline void ZUtils::object_copy_conjoint(uintptr_t from, uintptr_t to, size_t size) {
+inline void ZUtils::object_copy_conjoint(zaddress from, zaddress to, size_t size) {
   if (from != to) {
-    Copy::aligned_conjoint_words((HeapWord*)from, (HeapWord*)to, bytes_to_words(size));
+    Copy::aligned_conjoint_words((HeapWord*)untype(from), (HeapWord*)untype(to), bytes_to_words(size));
   }
 }
 
diff --git a/src/hotspot/share/gc/z/zValue.hpp b/src/hotspot/share/gc/z/zValue.hpp
index e2c67e8c48d..29f1b707e7c 100644
--- a/src/hotspot/share/gc/z/zValue.hpp
+++ b/src/hotspot/share/gc/z/zValue.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,7 @@
 #ifndef SHARE_GC_Z_ZVALUE_HPP
 #define SHARE_GC_Z_ZVALUE_HPP
 
+#include "memory/allocation.hpp"
 #include "memory/allStatic.hpp"
 #include "utilities/globalDefinitions.hpp"
 
diff --git a/src/hotspot/share/gc/z/zValue.inline.hpp b/src/hotspot/share/gc/z/zValue.inline.hpp
index ec574b8a09f..a5c905484e7 100644
--- a/src/hotspot/share/gc/z/zValue.inline.hpp
+++ b/src/hotspot/share/gc/z/zValue.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,10 +27,10 @@
 #include "gc/z/zValue.hpp"
 
 #include "gc/shared/gc_globals.hpp"
+#include "gc/shared/workerThread.hpp"
 #include "gc/z/zCPU.inline.hpp"
 #include "gc/z/zGlobals.hpp"
 #include "gc/z/zNUMA.hpp"
-#include "gc/z/zThread.inline.hpp"
 #include "gc/z/zUtils.hpp"
 #include "runtime/globals.hpp"
 #include "utilities/align.hpp"
@@ -106,11 +106,11 @@ inline size_t ZPerWorkerStorage::alignment() {
 }
 
 inline uint32_t ZPerWorkerStorage::count() {
-  return UseDynamicNumberOfGCThreads ? ConcGCThreads : MAX2(ConcGCThreads, ParallelGCThreads);
+  return ConcGCThreads;
 }
 
 inline uint32_t ZPerWorkerStorage::id() {
-  return ZThread::worker_id();
+  return WorkerThread::worker_id();
 }
 
 //
diff --git a/src/hotspot/share/gc/z/zVerify.cpp b/src/hotspot/share/gc/z/zVerify.cpp
index 689b0ded691..5742d46cb59 100644
--- a/src/hotspot/share/gc/z/zVerify.cpp
+++ b/src/hotspot/share/gc/z/zVerify.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -21,17 +21,20 @@
  * questions.
  */
 
+#include "memory/allocation.hpp"
 #include "precompiled.hpp"
 #include "classfile/classLoaderData.hpp"
 #include "gc/shared/gc_globals.hpp"
+#include "gc/shared/isGCActiveMark.hpp"
 #include "gc/z/zAddress.inline.hpp"
+#include "gc/z/zGenerationId.hpp"
 #include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zNMethod.hpp"
-#include "gc/z/zOop.hpp"
 #include "gc/z/zPageAllocator.hpp"
 #include "gc/z/zResurrection.hpp"
 #include "gc/z/zRootsIterator.hpp"
 #include "gc/z/zStackWatermark.hpp"
+#include "gc/z/zStoreBarrierBuffer.inline.hpp"
 #include "gc/z/zStat.hpp"
 #include "gc/z/zVerify.hpp"
 #include "memory/iterator.inline.hpp"
@@ -48,151 +51,199 @@
 #include "utilities/debug.hpp"
 #include "utilities/globalDefinitions.hpp"
 #include "utilities/preserveException.hpp"
+#include "utilities/resourceHash.hpp"
 
-#define BAD_OOP_ARG(o, p)   "Bad oop " PTR_FORMAT " found at " PTR_FORMAT, p2i(o), p2i(p)
+#define BAD_OOP_ARG(o, p)   "Bad oop " PTR_FORMAT " found at " PTR_FORMAT, untype(o), p2i(p)
 
-static void z_verify_oop(oop* p) {
-  const oop o = RawAccess<>::oop_load(p);
-  if (o != NULL) {
-    const uintptr_t addr = ZOop::to_address(o);
-    guarantee(ZAddress::is_good(addr), BAD_OOP_ARG(o, p));
-    guarantee(oopDesc::is_oop(ZOop::from_address(addr)), BAD_OOP_ARG(o, p));
+static bool z_is_null_relaxed(zpointer o) {
+  const uintptr_t color_mask = ZPointerAllMetadataMask | ZPointerReservedMask;
+  return (untype(o) & ~color_mask) == 0;
+}
+
+static void z_verify_old_oop(zpointer* p) {
+  const zpointer o = *p;
+  assert(o != zpointer::null, "Old should not contain raw null");
+  if (!z_is_null_relaxed(o)) {
+    if (ZPointer::is_mark_good(o)) {
+      // Even though the pointer is mark good, we can't verify that it should
+      // be in the remembered set in old mark end. We have to wait to the verify
+      // safepoint after reference processing, where we hold the driver lock and
+      // know there is no concurrent remembered set processing in the young generation.
+      const zaddress addr = ZPointer::uncolor(o);
+      guarantee(oopDesc::is_oop(to_oop(addr)), BAD_OOP_ARG(o, p));
+    } else {
+      const zaddress addr = ZBarrier::load_barrier_on_oop_field_preloaded(nullptr, o);
+      // Old to young pointers might not be mark good if the young
+      // marking has not finished, which is responsible for coloring
+      // these pointers.
+      if (ZHeap::heap()->is_old(addr) || !ZGeneration::young()->is_phase_mark()) {
+        // Old to old pointers are allowed to have bad young bits
+        guarantee(ZPointer::is_marked_old(o),  BAD_OOP_ARG(o, p));
+        guarantee(ZHeap::heap()->is_old(p), BAD_OOP_ARG(o, p));
+      }
+    }
   }
 }
 
-static void z_verify_possibly_weak_oop(oop* p) {
-  const oop o = RawAccess<>::oop_load(p);
-  if (o != NULL) {
-    const uintptr_t addr = ZOop::to_address(o);
-    guarantee(ZAddress::is_good(addr) || ZAddress::is_finalizable_good(addr), BAD_OOP_ARG(o, p));
-    guarantee(oopDesc::is_oop(ZOop::from_address(ZAddress::good(addr))), BAD_OOP_ARG(o, p));
+static void z_verify_young_oop(zpointer* p) {
+  const zpointer o = *p;
+  if (!z_is_null_relaxed(o)) {
+    guarantee(ZHeap::heap()->is_young(p), BAD_OOP_ARG(o, p));
+    guarantee(ZPointer::is_marked_young(o),  BAD_OOP_ARG(o, p));
+
+    if (ZPointer::is_load_good(o)) {
+      guarantee(oopDesc::is_oop(to_oop(ZPointer::uncolor(o))), BAD_OOP_ARG(o, p));
+    }
   }
 }
 
-class ZVerifyRootClosure : public OopClosure {
+static void z_verify_root_oop_object(zaddress o, void* p) {
+  guarantee(oopDesc::is_oop(to_oop(o)), BAD_OOP_ARG(o, p));
+}
+
+static void z_verify_uncolored_root_oop(zaddress* p) {
+  assert(!ZHeap::heap()->is_in((uintptr_t)p), "Roots shouldn't be in heap");
+  const zaddress o = *p;
+  if (!is_null(o)) {
+    z_verify_root_oop_object(o, p);
+  }
+}
+
+static void z_verify_possibly_weak_oop(zpointer* p) {
+  const zpointer o = *p;
+  if (!z_is_null_relaxed(o)) {
+    guarantee(ZPointer::is_marked_old(o) || ZPointer::is_marked_finalizable(o), BAD_OOP_ARG(o, p));
+
+    const zaddress addr = ZBarrier::load_barrier_on_oop_field_preloaded(nullptr, o);
+    guarantee(ZHeap::heap()->is_old(addr) || ZPointer::is_marked_young(o), BAD_OOP_ARG(o, p));
+    guarantee(ZHeap::heap()->is_young(addr) || ZHeap::heap()->is_object_live(addr), BAD_OOP_ARG(o, p));
+    guarantee(oopDesc::is_oop(to_oop(addr)), BAD_OOP_ARG(o, p));
+
+    // Verify no missing remset entries. We are holding the driver lock here and that
+    // allows us to more precisely verify the remembered set, as there is no concurrent
+    // young generation collection going on at this point.
+    const uintptr_t remset_bits = untype(o) & ZPointerRememberedMask;
+    const uintptr_t prev_remembered = ZPointerRemembered ^ ZPointerRememberedMask;
+    guarantee(remset_bits != prev_remembered, BAD_OOP_ARG(o, p));
+    guarantee(remset_bits == ZPointerRememberedMask ||
+              ZGeneration::young()->is_remembered(p) ||
+              ZStoreBarrierBuffer::is_in(p), BAD_OOP_ARG(o, p));
+  }
+}
+
+class ZVerifyColoredRootClosure : public OopClosure {
 private:
-  const bool _verify_fixed;
+  const bool _verify_marked_old;
 
 public:
-  ZVerifyRootClosure(bool verify_fixed) :
-      _verify_fixed(verify_fixed) {}
+  ZVerifyColoredRootClosure(bool verify_marked_old) :
+      OopClosure(),
+      _verify_marked_old(verify_marked_old) {}
 
-  virtual void do_oop(oop* p) {
-    if (_verify_fixed) {
-      z_verify_oop(p);
+  virtual void do_oop(oop* p_) {
+    zpointer* const p = (zpointer*)p_;
+
+    assert(!ZHeap::heap()->is_in((uintptr_t)p), "Roots shouldn't be in heap");
+
+    const zpointer o = *p;
+
+    if (z_is_null_relaxed(o)) {
+      // Skip verifying nulls
+      return;
+    }
+
+    assert(is_valid(o), "Catch me!");
+
+    if (_verify_marked_old) {
+      guarantee(ZPointer::is_marked_old(o), BAD_OOP_ARG(o, p));
+
+      // Minor collections could have relocated the object;
+      // use load barrier to find correct object.
+      const zaddress addr = ZBarrier::load_barrier_on_oop_field_preloaded(nullptr, o);
+      z_verify_root_oop_object(addr, p);
     } else {
-      // Don't know the state of the oop.
-      oop obj = *p;
-      obj = NativeAccess<AS_NO_KEEPALIVE>::oop_load(&obj);
-      z_verify_oop(&obj);
+      // Don't know the state of the oop
+      if (is_valid(o)) {
+        // it looks like a valid colored oop;
+        // use load barrier to find correct object.
+        const zaddress addr = ZBarrier::load_barrier_on_oop_field_preloaded(nullptr, o);
+        z_verify_root_oop_object(addr, p);
+      }
     }
   }
 
   virtual void do_oop(narrowOop*) {
     ShouldNotReachHere();
   }
+};
 
-  bool verify_fixed() const {
-    return _verify_fixed;
+class ZVerifyUncoloredRootClosure : public OopClosure {
+public:
+  virtual void do_oop(oop* p_) {
+    zaddress* const p = (zaddress*)p_;
+    z_verify_uncolored_root_oop(p);
+  }
+
+  virtual void do_oop(narrowOop*) {
+    ShouldNotReachHere();
   }
 };
 
 class ZVerifyCodeBlobClosure : public CodeBlobToOopClosure {
 public:
-  ZVerifyCodeBlobClosure(ZVerifyRootClosure* _cl) :
-      CodeBlobToOopClosure(_cl, false /* fix_relocations */) {}
+  ZVerifyCodeBlobClosure(OopClosure* cl) :
+      CodeBlobToOopClosure(cl, false /* fix_relocations */) {}
 
   virtual void do_code_blob(CodeBlob* cb) {
     CodeBlobToOopClosure::do_code_blob(cb);
   }
 };
 
-class ZVerifyStack : public OopClosure {
-private:
-  ZVerifyRootClosure* const _cl;
-  JavaThread*         const _jt;
-  uint64_t                  _last_good;
-  bool                      _verifying_bad_frames;
-
-public:
-  ZVerifyStack(ZVerifyRootClosure* cl, JavaThread* jt) :
-      _cl(cl),
-      _jt(jt),
-      _last_good(0),
-      _verifying_bad_frames(false) {
-    ZStackWatermark* const stack_watermark = StackWatermarkSet::get<ZStackWatermark>(jt, StackWatermarkKind::gc);
-
-    if (_cl->verify_fixed()) {
-      assert(stack_watermark->processing_started(), "Should already have been fixed");
-      assert(stack_watermark->processing_completed(), "Should already have been fixed");
-    } else {
-      // We don't really know the state of the stack, verify watermark.
-      if (!stack_watermark->processing_started()) {
-        _verifying_bad_frames = true;
-      } else {
-        // Not time yet to verify bad frames
-        _last_good = stack_watermark->last_processed();
-      }
-    }
-  }
-
-  void do_oop(oop* p) {
-    if (_verifying_bad_frames) {
-      const oop obj = *p;
-      guarantee(!ZAddress::is_good(ZOop::to_address(obj)), BAD_OOP_ARG(obj, p));
-    }
-    _cl->do_oop(p);
-  }
-
-  void do_oop(narrowOop* p) {
-    ShouldNotReachHere();
-  }
-
-  void prepare_next_frame(frame& frame) {
-    if (_cl->verify_fixed()) {
-      // All frames need to be good
-      return;
-    }
-
-    // The verification has two modes, depending on whether we have reached the
-    // last processed frame or not. Before it is reached, we expect everything to
-    // be good. After reaching it, we expect everything to be bad.
-    const uintptr_t sp = reinterpret_cast<uintptr_t>(frame.sp());
-
-    if (!_verifying_bad_frames && sp == _last_good) {
-      // Found the last good frame, now verify the bad ones
-      _verifying_bad_frames = true;
-    }
-  }
-
-  void verify_frames() {
-    ZVerifyCodeBlobClosure cb_cl(_cl);
-    for (StackFrameStream frames(_jt, true /* update */, false /* process_frames */);
-         !frames.is_done();
-         frames.next()) {
-      frame& frame = *frames.current();
-      frame.oops_do(this, &cb_cl, frames.register_map(), DerivedPointerIterationMode::_ignore);
-      prepare_next_frame(frame);
-    }
-  }
-};
-
-class ZVerifyOopClosure : public ClaimMetadataVisitingOopIterateClosure {
+class ZVerifyOldOopClosure : public BasicOopIterateClosure {
 private:
   const bool _verify_weaks;
 
 public:
-  ZVerifyOopClosure(bool verify_weaks) :
-      ClaimMetadataVisitingOopIterateClosure(ClassLoaderData::_claim_other),
+  ZVerifyOldOopClosure(bool verify_weaks) :
       _verify_weaks(verify_weaks) {}
 
-  virtual void do_oop(oop* p) {
+  virtual void do_oop(oop* p_) {
+    zpointer* const p = (zpointer*)p_;
     if (_verify_weaks) {
       z_verify_possibly_weak_oop(p);
     } else {
       // We should never encounter finalizable oops through strong
       // paths. This assumes we have only visited strong roots.
-      z_verify_oop(p);
+      z_verify_old_oop(p);
+    }
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+
+  virtual ReferenceIterationMode reference_iteration_mode() {
+    return _verify_weaks ? DO_FIELDS : DO_FIELDS_EXCEPT_REFERENT;
+  }
+};
+
+class ZVerifyYoungOopClosure : public BasicOopIterateClosure {
+private:
+  const bool _verify_weaks;
+
+public:
+  ZVerifyYoungOopClosure(bool verify_weaks) :
+      _verify_weaks(verify_weaks) {}
+
+  virtual void do_oop(oop* p_) {
+    zpointer* const p = (zpointer*)p_;
+    if (_verify_weaks) {
+      //z_verify_possibly_weak_oop(p);
+      z_verify_young_oop(p);
+    } else {
+      // We should never encounter finalizable oops through strong
+      // paths. This assumes we have only visited strong roots.
+      z_verify_young_oop(p);
     }
   }
 
@@ -213,22 +264,22 @@ typedef ClaimingCLDToOopClosure<ClassLoaderData::_claim_none> ZVerifyCLDClosure;
 
 class ZVerifyThreadClosure : public ThreadClosure {
 private:
-  ZVerifyRootClosure* const _cl;
+  OopClosure* const _verify_cl;
 
 public:
-  ZVerifyThreadClosure(ZVerifyRootClosure* cl) :
-      _cl(cl) {}
+  ZVerifyThreadClosure(OopClosure* verify_cl) :
+      _verify_cl(verify_cl) {}
 
   virtual void do_thread(Thread* thread) {
-    thread->oops_do_no_frames(_cl, NULL);
-
     JavaThread* const jt = JavaThread::cast(thread);
-    if (!jt->has_last_Java_frame()) {
-      return;
-    }
+    const ZStackWatermark* const watermark = StackWatermarkSet::get<ZStackWatermark>(jt, StackWatermarkKind::gc);
+    if (watermark->processing_started_acquire()) {
+      thread->oops_do_no_frames(_verify_cl, nullptr);
 
-    ZVerifyStack verify_stack(_cl, jt);
-    verify_stack.verify_frames();
+       if (watermark->processing_completed_acquire()) {
+         thread->oops_do_frames(_verify_cl, nullptr);
+       }
+    }
   }
 };
 
@@ -236,86 +287,173 @@ class ZVerifyNMethodClosure : public NMethodClosure {
 private:
   OopClosure* const        _cl;
   BarrierSetNMethod* const _bs_nm;
-  const bool               _verify_fixed;
-
-  bool trust_nmethod_state() const {
-    // The root iterator will visit non-processed
-    // nmethods class unloading is turned off.
-    return ClassUnloading || _verify_fixed;
-  }
 
 public:
-  ZVerifyNMethodClosure(OopClosure* cl, bool verify_fixed) :
+  ZVerifyNMethodClosure(OopClosure* cl) :
       _cl(cl),
-      _bs_nm(BarrierSet::barrier_set()->barrier_set_nmethod()),
-      _verify_fixed(verify_fixed) {}
+      _bs_nm(BarrierSet::barrier_set()->barrier_set_nmethod()) {}
 
   virtual void do_nmethod(nmethod* nm) {
-    assert(!trust_nmethod_state() || !_bs_nm->is_armed(nm), "Should not encounter any armed nmethods");
+    if (_bs_nm->is_armed(nm)) {
+      // Can't verify
+      return;
+    }
 
     ZNMethod::nmethod_oops_do(nm, _cl);
   }
 };
 
-void ZVerify::roots_strong(bool verify_fixed) {
+void ZVerify::roots_strong(bool verify_after_old_mark) {
   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
-  assert(!ZResurrection::is_blocked(), "Invalid phase");
 
-  ZVerifyRootClosure cl(verify_fixed);
-  ZVerifyCLDClosure cld_cl(&cl);
-  ZVerifyThreadClosure thread_cl(&cl);
-  ZVerifyNMethodClosure nm_cl(&cl, verify_fixed);
+  {
+    ZVerifyColoredRootClosure cl(verify_after_old_mark);
+    ZVerifyCLDClosure cld_cl(&cl);
 
-  ZRootsIterator iter(ClassLoaderData::_claim_none);
-  iter.apply(&cl,
-             &cld_cl,
-             &thread_cl,
-             &nm_cl);
+    ZRootsIteratorStrongColored roots_strong_colored(ZGenerationIdOptional::none);
+    roots_strong_colored.apply(&cl,
+                               &cld_cl);
+  }
+
+  {
+    ZVerifyUncoloredRootClosure cl;
+    ZVerifyThreadClosure thread_cl(&cl);
+    ZVerifyNMethodClosure nm_cl(&cl);
+
+    ZRootsIteratorStrongUncolored roots_strong_uncolored(ZGenerationIdOptional::none);
+    roots_strong_uncolored.apply(&thread_cl,
+                                 &nm_cl);
+  }
 }
 
 void ZVerify::roots_weak() {
   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
   assert(!ZResurrection::is_blocked(), "Invalid phase");
 
-  ZVerifyRootClosure cl(true /* verify_fixed */);
-  ZWeakRootsIterator iter;
-  iter.apply(&cl);
+  ZVerifyColoredRootClosure cl(true /* verify_after_old_mark*/);
+  ZRootsIteratorWeakColored roots_weak_colored(ZGenerationIdOptional::none);
+  roots_weak_colored.apply(&cl);
+}
+
+zaddress zverify_broken_object = zaddress::null;
+
+class ZVerifyObjectClosure : public ObjectClosure, public OopFieldClosure {
+private:
+  const bool         _verify_weaks;
+
+  zaddress           _visited_base;
+  volatile zpointer* _visited_p;
+  zpointer           _visited_ptr_pre_loaded;
+
+public:
+  ZVerifyObjectClosure(bool verify_weaks) :
+      _verify_weaks(verify_weaks),
+      _visited_base(),
+      _visited_p(),
+      _visited_ptr_pre_loaded() {}
+
+  void log_dead_object(zaddress addr) {
+    tty->print_cr("ZVerify found dead object: " PTR_FORMAT " at p: " PTR_FORMAT " ptr: " PTR_FORMAT, untype(addr), p2i((void*)_visited_p), untype(_visited_ptr_pre_loaded));
+    to_oop(addr)->print();
+    tty->print_cr("--- From --- ");
+    if (_visited_base != zaddress::null) {
+      to_oop(_visited_base)->print();
+    }
+    tty->cr();
+
+    if (zverify_broken_object == zaddress::null) {
+      zverify_broken_object = addr;
+    }
+  }
+
+  void verify_live_object(oop obj) {
+    // Verify that its pointers are sane
+    ZVerifyOldOopClosure cl(_verify_weaks);
+    ZIterator::oop_iterate_safe(obj, &cl);
+  }
+
+  virtual void do_object(oop obj) {
+    guarantee(oopDesc::is_oop_or_null(obj), "Must be");
+
+    const zaddress addr = to_zaddress(obj);
+    if (ZHeap::heap()->is_old(addr)) {
+      if (ZHeap::heap()->is_object_live(addr)) {
+        verify_live_object(obj);
+      } else {
+        log_dead_object(addr);
+      }
+    } else {
+      // Young object - no verification
+    }
+  }
+
+  virtual void do_field(oop base, oop* p) {
+    _visited_base = to_zaddress(base);
+    _visited_p = (volatile zpointer*)p;
+    _visited_ptr_pre_loaded = Atomic::load(_visited_p);
+  }
+};
+
+void ZVerify::threads_start_processing() {
+  class StartProcessingClosure : public ThreadClosure {
+  public:
+    void do_thread(Thread* thread) {
+      StackWatermarkSet::start_processing(JavaThread::cast(thread), StackWatermarkKind::gc);
+    }
+  };
+
+  ZJavaThreadsIterator threads_iterator(ZGenerationIdOptional::none);
+  StartProcessingClosure cl;
+  threads_iterator.apply(&cl);
 }
 
 void ZVerify::objects(bool verify_weaks) {
+  if (ZAbort::should_abort()) {
+    // Invariants might be a bit mushy if the young generation
+    // collection was forced to shut down. So let's be a bit forgiving here.
+    return;
+  }
   assert(SafepointSynchronize::is_at_safepoint(), "Must be at a safepoint");
-  assert(ZGlobalPhase == ZPhaseMarkCompleted, "Invalid phase");
+  assert(ZGeneration::young()->is_phase_mark_complete() ||
+         ZGeneration::old()->is_phase_mark_complete(), "Invalid phase");
   assert(!ZResurrection::is_blocked(), "Invalid phase");
 
-  ZVerifyOopClosure cl(verify_weaks);
-  ObjectToOopClosure object_cl(&cl);
-  ZHeap::heap()->object_iterate(&object_cl, verify_weaks);
+  // Note that object verification will fix the pointers and
+  // only verify that the resulting objects are sane.
+
+  // The verification VM_Operation doesn't start the thread processing.
+  // Do it here, after the roots have been verified.
+  threads_start_processing();
+
+  ZVerifyObjectClosure object_cl(verify_weaks);
+  ZHeap::heap()->object_and_field_iterate(&object_cl, &object_cl, verify_weaks);
 }
 
 void ZVerify::before_zoperation() {
   // Verify strong roots
-  ZStatTimerDisable disable;
   if (ZVerifyRoots) {
-    roots_strong(false /* verify_fixed */);
+    roots_strong(false /* verify_after_old_mark */);
   }
 }
 
 void ZVerify::after_mark() {
   // Verify all strong roots and strong references
-  ZStatTimerDisable disable;
   if (ZVerifyRoots) {
-    roots_strong(true /* verify_fixed */);
+    roots_strong(true /* verify_after_old_mark */);
   }
   if (ZVerifyObjects) {
+    // Workaround OopMapCacheAlloc_lock reordering with the StackWatermark_lock
+    DisableIsGCActiveMark mark;
+
     objects(false /* verify_weaks */);
+    guarantee(zverify_broken_object == zaddress::null, "Verification failed");
   }
 }
 
 void ZVerify::after_weak_processing() {
   // Verify all roots and all references
-  ZStatTimerDisable disable;
   if (ZVerifyRoots) {
-    roots_strong(true /* verify_fixed */);
+    roots_strong(true /* verify_after_old_mark */);
     roots_weak();
   }
   if (ZVerifyObjects) {
@@ -323,93 +461,268 @@ void ZVerify::after_weak_processing() {
   }
 }
 
-template <bool Map>
-class ZPageDebugMapOrUnmapClosure : public ZPageClosure {
+//
+// Remembered set verification
+//
+
+typedef ResourceHashtable<volatile zpointer*, bool, 1009, AnyObj::C_HEAP, mtGC> ZStoreBarrierBufferTable;
+
+static ZStoreBarrierBufferTable* z_verify_store_barrier_buffer_table = nullptr;
+
+#define BAD_REMSET_ARG(p, ptr, addr) \
+  "Missing remembered set at " PTR_FORMAT " pointing at " PTR_FORMAT \
+  " (" PTR_FORMAT " + " INTX_FORMAT ")" \
+  , p2i(p), untype(ptr), untype(addr), p2i(p) - untype(addr)
+
+class ZVerifyRemsetBeforeOopClosure : public BasicOopIterateClosure {
 private:
-  const ZPageAllocator* const _allocator;
+  ZForwarding*    _forwarding;
+  zaddress_unsafe _from_addr;
 
 public:
-  ZPageDebugMapOrUnmapClosure(const ZPageAllocator* allocator) :
-      _allocator(allocator) {}
+  ZVerifyRemsetBeforeOopClosure(ZForwarding* forwarding) :
+      _forwarding(forwarding),
+      _from_addr(zaddress_unsafe::null) {}
 
-  void do_page(const ZPage* page) {
-    if (Map) {
-      _allocator->debug_map_page(page);
-    } else {
-      _allocator->debug_unmap_page(page);
+  void set_from_addr(zaddress_unsafe addr) {
+    _from_addr = addr;
+  }
+
+  virtual void do_oop(oop* p_) {
+    volatile zpointer* const p = (volatile zpointer*)p_;
+    const zpointer ptr = *p;
+
+    if (ZPointer::is_remembered_exact(ptr)) {
+      // When the remembered bits are 11, it means that it is intentionally
+      // not part of the remembered set
+      return;
     }
-  }
-};
 
-ZVerifyViewsFlip::ZVerifyViewsFlip(const ZPageAllocator* allocator) :
-    _allocator(allocator) {
-  if (ZVerifyViews) {
-    // Unmap all pages
-    ZPageDebugMapOrUnmapClosure<false /* Map */> cl(_allocator);
-    ZHeap::heap()->pages_do(&cl);
-  }
-}
+    if (ZBufferStoreBarriers && z_verify_store_barrier_buffer_table->get(p) != nullptr) {
+      // If this oop location is in the store barrier buffer, we can't assume
+      // that it should have a remset entry
+      return;
+    }
 
-ZVerifyViewsFlip::~ZVerifyViewsFlip() {
-  if (ZVerifyViews) {
-    // Map all pages
-    ZPageDebugMapOrUnmapClosure<true /* Map */> cl(_allocator);
-    ZHeap::heap()->pages_do(&cl);
-  }
-}
+    if (_forwarding->find(_from_addr) != zaddress::null) {
+      // If the mutator has already relocated the object to to-space, we defer
+      // and do to-space verification afterwards instead, because store barrier
+      // buffers could have installed the remembered set entry in to-space and
+      // then flushed the store barrier buffer, and then start young marking
+      return;
+    }
 
-#ifdef ASSERT
+    ZPage* page = _forwarding->page();
 
-class ZVerifyBadOopClosure : public OopClosure {
-public:
-  virtual void do_oop(oop* p) {
-    const oop o = *p;
-    assert(!ZAddress::is_good(ZOop::to_address(o)), "Should not be good: " PTR_FORMAT, p2i(o));
+    if (ZGeneration::old()->active_remset_is_current()) {
+      guarantee(page->is_remembered(p), BAD_REMSET_ARG(p, ptr, _from_addr));
+    } else {
+      guarantee(page->was_remembered(p), BAD_REMSET_ARG(p, ptr, _from_addr));
+    }
   }
 
   virtual void do_oop(narrowOop* p) {
     ShouldNotReachHere();
   }
+
+  virtual ReferenceIterationMode reference_iteration_mode() {
+    return DO_FIELDS;
+  }
 };
 
-// This class encapsulates various marks we need to deal with calling the
-// frame iteration code from arbitrary points in the runtime. It is mostly
-// due to problems that we might want to eventually clean up inside of the
-// frame iteration code, such as creating random handles even though there
-// is no safepoint to protect against, and fiddling around with exceptions.
-class StackWatermarkProcessingMark {
-  ResetNoHandleMark     _rnhm;
-  HandleMark            _hm;
-  PreserveExceptionMark _pem;
-  ResourceMark          _rm;
+void ZVerify::on_color_flip() {
+  if (!ZVerifyRemembered || !ZBufferStoreBarriers) {
+    return;
+  }
 
-public:
-  StackWatermarkProcessingMark(Thread* thread) :
-      _rnhm(),
-      _hm(thread),
-      _pem(thread),
-      _rm(thread) {}
-};
+  // Reset the table tracking the stale stores of the store barrier buffer
+  delete z_verify_store_barrier_buffer_table;
+  z_verify_store_barrier_buffer_table = new (mtGC) ZStoreBarrierBufferTable();
 
-void ZVerify::verify_frame_bad(const frame& fr, RegisterMap& register_map) {
-  ZVerifyBadOopClosure verify_cl;
-  fr.oops_do(&verify_cl, NULL, &register_map, DerivedPointerIterationMode::_ignore);
-}
+  // Gather information from store barrier buffers as we currently can't verify
+  // remset entries for oop locations touched by the store barrier buffer
 
-void ZVerify::verify_thread_head_bad(JavaThread* jt) {
-  ZVerifyBadOopClosure verify_cl;
-  jt->oops_do_no_frames(&verify_cl, NULL);
-}
+  for (JavaThreadIteratorWithHandle jtiwh; JavaThread* const jt = jtiwh.next(); ) {
+    const ZStoreBarrierBuffer* const buffer = ZThreadLocalData::store_barrier_buffer(jt);
 
-void ZVerify::verify_thread_frames_bad(JavaThread* jt) {
-  if (jt->has_last_Java_frame()) {
-    ZVerifyBadOopClosure verify_cl;
-    StackWatermarkProcessingMark swpm(Thread::current());
-    // Traverse the execution stack
-    for (StackFrameStream fst(jt, true /* update */, false /* process_frames */); !fst.is_done(); fst.next()) {
-      fst.current()->oops_do(&verify_cl, NULL /* code_cl */, fst.register_map(), DerivedPointerIterationMode::_ignore);
+    for (int i = buffer->current(); i < (int)ZStoreBarrierBuffer::_buffer_length; ++i) {
+      volatile zpointer* const p = buffer->_buffer[i]._p;
+      bool created = false;
+      z_verify_store_barrier_buffer_table->put_if_absent(p, true, &created);
     }
   }
 }
 
-#endif // ASSERT
+void ZVerify::before_relocation(ZForwarding* forwarding) {
+  if (!ZVerifyRemembered) {
+    return;
+  }
+
+  if (forwarding->from_age() != ZPageAge::old) {
+    // Only supports verification of old-to-old relocations now
+    return;
+  }
+
+  // Verify that the inactive remset is cleared
+  if (ZGeneration::old()->active_remset_is_current()) {
+    forwarding->page()->verify_remset_cleared_previous();
+  } else {
+    forwarding->page()->verify_remset_cleared_current();
+  }
+
+  ZVerifyRemsetBeforeOopClosure cl(forwarding);
+
+  forwarding->object_iterate([&](oop obj) {
+    const zaddress_unsafe addr = to_zaddress_unsafe(cast_from_oop<uintptr_t>(obj));
+    cl.set_from_addr(addr);
+    obj->oop_iterate(&cl);
+  });
+}
+
+class ZVerifyRemsetAfterOopClosure : public BasicOopIterateClosure {
+private:
+  ZForwarding* const _forwarding;
+  zaddress_unsafe    _from_addr;
+  zaddress           _to_addr;
+
+public:
+  ZVerifyRemsetAfterOopClosure(ZForwarding* forwarding) :
+      _forwarding(forwarding),
+      _from_addr(zaddress_unsafe::null),
+      _to_addr(zaddress::null) {}
+
+  void set_from_addr(zaddress_unsafe addr) {
+    _from_addr = addr;
+  }
+
+  void set_to_addr(zaddress addr) {
+    _to_addr = addr;
+  }
+
+  virtual void do_oop(oop* p_) {
+    volatile zpointer* const p = (volatile zpointer*)p_;
+    const zpointer ptr = Atomic::load(p);
+
+    // Order this load w.r.t. the was_remembered load which can race when
+    // the remset scanning of the to-space object is concurrently forgetting
+    // an entry.
+    OrderAccess::loadload();
+
+    if (ZPointer::is_remembered_exact(ptr)) {
+      // When the remembered bits are 11, it means that it is intentionally
+      // not part of the remembered set
+      return;
+    }
+
+    if (ZPointer::is_store_good(ptr)) {
+      // In to-space, there could be stores racing with the verification.
+      // Such stores may not have reliably manifested in the remembered
+      // sets yet.
+      return;
+    }
+
+    if (ZBufferStoreBarriers && z_verify_store_barrier_buffer_table->get(p) != nullptr) {
+      // If this to-space oop location is in the store barrier buffer, we
+      // can't assume that it should have a remset entry
+      return;
+    }
+
+    const uintptr_t p_offset = uintptr_t(p) - untype(_to_addr);
+    volatile zpointer* const fromspace_p = (volatile zpointer*)(untype(_from_addr) + p_offset);
+
+    if (ZBufferStoreBarriers && z_verify_store_barrier_buffer_table->get(fromspace_p) != nullptr) {
+      // If this from-space oop location is in the store barrier buffer, we
+      // can't assume that it should have a remset entry
+      return;
+    }
+
+    ZPage* page = ZHeap::heap()->page(p);
+
+    if (page->is_remembered(p) || page->was_remembered(p)) {
+      // No missing remembered set entry
+      return;
+    }
+
+    OrderAccess::loadload();
+    if (Atomic::load(p) != ptr) {
+      // Order the was_remembered bitmap load w.r.t. the reload of the zpointer.
+      // Sometimes the was_remembered() call above races with clearing of the
+      // previous bits, when the to-space object is concurrently forgetting
+      // remset entries because they were not so useful. When that happens,
+      // we have already self healed the pointers to have 11 in the remset
+      // bits.
+      return;
+    }
+
+    guarantee(ZGeneration::young()->is_phase_mark(), "Should be in the mark phase " BAD_REMSET_ARG(p, ptr, _to_addr));
+    guarantee(_forwarding->relocated_remembered_fields_published_contains(p), BAD_REMSET_ARG(p, ptr, _to_addr));
+  }
+
+  virtual void do_oop(narrowOop* p) {
+    ShouldNotReachHere();
+  }
+
+  virtual ReferenceIterationMode reference_iteration_mode() {
+    return DO_FIELDS;
+  }
+};
+
+void ZVerify::after_relocation_internal(ZForwarding* forwarding) {
+  ZVerifyRemsetAfterOopClosure cl(forwarding);
+
+  forwarding->address_unsafe_iterate_via_table([&](zaddress_unsafe from_addr) {
+    // If no field in this object was in the store barrier buffer
+    // when relocation started, we should be able to verify trivially
+    ZGeneration* const from_generation = forwarding->from_age() == ZPageAge::old ? (ZGeneration*)ZGeneration::old()
+                                                                           : (ZGeneration*)ZGeneration::young();
+    const zaddress to_addr = from_generation->remap_object(from_addr);
+
+    cl.set_from_addr(from_addr);
+    cl.set_to_addr(to_addr);
+    const oop to_obj = to_oop(to_addr);
+    to_obj->oop_iterate(&cl);
+  });
+}
+
+void ZVerify::after_relocation(ZForwarding* forwarding) {
+  if (!ZVerifyRemembered) {
+    return;
+  }
+
+  if (forwarding->to_age() != ZPageAge::old) {
+    // No remsets to verify in the young gen
+    return;
+  }
+
+  if (ZGeneration::young()->is_phase_mark() &&
+      forwarding->relocated_remembered_fields_is_concurrently_scanned()) {
+    // Can't verify to-space objects if concurrent YC rejected published
+    // remset information, because that data is incomplete. The YC might
+    // not have finished scanning the forwarding, and might be about to
+    // insert required remembered set entries.
+    return;
+  }
+
+  after_relocation_internal(forwarding);
+}
+
+void ZVerify::after_scan(ZForwarding* forwarding) {
+  if (!ZVerifyRemembered) {
+    return;
+  }
+
+  if (ZAbort::should_abort()) {
+    // We can't verify remembered set accurately when shutting down the VM
+    return;
+  }
+
+  if (!ZGeneration::old()->is_phase_relocate() ||
+      !forwarding->relocated_remembered_fields_is_concurrently_scanned()) {
+    // Only verify remembered set from remembered set scanning, when the
+    // remembered set scanning rejected the publishing information of concurrent
+    // old generation relocation
+    return;
+  }
+
+  after_relocation_internal(forwarding);
+}
diff --git a/src/hotspot/share/gc/z/zVerify.hpp b/src/hotspot/share/gc/z/zVerify.hpp
index 8d7abd4a8d5..e9ada2cefa9 100644
--- a/src/hotspot/share/gc/z/zVerify.hpp
+++ b/src/hotspot/share/gc/z/zVerify.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,32 +27,29 @@
 #include "memory/allStatic.hpp"
 
 class frame;
+class ZForwarding;
 class ZPageAllocator;
 
 class ZVerify : public AllStatic {
 private:
-  static void roots_strong(bool verify_fixed);
+  static void roots_strong(bool verify_after_old_mark);
   static void roots_weak();
 
   static void objects(bool verify_weaks);
+  static void threads_start_processing();
+
+  static void after_relocation_internal(ZForwarding* forwarding);
 
 public:
   static void before_zoperation();
   static void after_mark();
   static void after_weak_processing();
 
-  static void verify_thread_head_bad(JavaThread* thread) NOT_DEBUG_RETURN;
-  static void verify_thread_frames_bad(JavaThread* thread) NOT_DEBUG_RETURN;
-  static void verify_frame_bad(const frame& fr, RegisterMap& register_map) NOT_DEBUG_RETURN;
-};
+  static void before_relocation(ZForwarding* forwarding);
+  static void after_relocation(ZForwarding* forwarding);
+  static void after_scan(ZForwarding* forwarding);
 
-class ZVerifyViewsFlip {
-private:
-  const ZPageAllocator* const _allocator;
-
-public:
-  ZVerifyViewsFlip(const ZPageAllocator* allocator);
-  ~ZVerifyViewsFlip();
+  static void on_color_flip();
 };
 
 #endif // SHARE_GC_Z_ZVERIFY_HPP
diff --git a/src/hotspot/share/gc/z/zVirtualMemory.cpp b/src/hotspot/share/gc/z/zVirtualMemory.cpp
index 9366412cea2..7187c1cac94 100644
--- a/src/hotspot/share/gc/z/zVirtualMemory.cpp
+++ b/src/hotspot/share/gc/z/zVirtualMemory.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -36,12 +36,7 @@ ZVirtualMemoryManager::ZVirtualMemoryManager(size_t max_capacity) :
     _reserved(0),
     _initialized(false) {
 
-  // Check max supported heap size
-  if (max_capacity > ZAddressOffsetMax) {
-    log_error_p(gc)("Java heap too large (max supported heap size is " SIZE_FORMAT "G)",
-                    ZAddressOffsetMax / G);
-    return;
-  }
+  assert(max_capacity <= ZAddressOffsetMax, "Too large max_capacity");
 
   // Initialize platform specific parts before reserving address space
   pd_initialize_before_reserve();
@@ -59,7 +54,7 @@ ZVirtualMemoryManager::ZVirtualMemoryManager(size_t max_capacity) :
   _initialized = true;
 }
 
-size_t ZVirtualMemoryManager::reserve_discontiguous(uintptr_t start, size_t size, size_t min_range) {
+size_t ZVirtualMemoryManager::reserve_discontiguous(zoffset start, size_t size, size_t min_range) {
   if (size < min_range) {
     // Too small
     return 0;
@@ -89,47 +84,32 @@ size_t ZVirtualMemoryManager::reserve_discontiguous(size_t size) {
   // This avoids an explosion of reservation attempts in case large parts of the
   // address space is already occupied.
   const size_t min_range = align_up(size / 100, ZGranuleSize);
-  size_t start = 0;
+  uintptr_t start = 0;
   size_t reserved = 0;
 
   // Reserve size somewhere between [0, ZAddressOffsetMax)
   while (reserved < size && start < ZAddressOffsetMax) {
     const size_t remaining = MIN2(size - reserved, ZAddressOffsetMax - start);
-    reserved += reserve_discontiguous(start, remaining, min_range);
+    reserved += reserve_discontiguous(to_zoffset(start), remaining, min_range);
     start += remaining;
   }
 
   return reserved;
 }
 
-bool ZVirtualMemoryManager::reserve_contiguous(uintptr_t start, size_t size) {
+bool ZVirtualMemoryManager::reserve_contiguous(zoffset start, size_t size) {
   assert(is_aligned(size, ZGranuleSize), "Must be granule aligned");
 
   // Reserve address views
-  const uintptr_t marked0 = ZAddress::marked0(start);
-  const uintptr_t marked1 = ZAddress::marked1(start);
-  const uintptr_t remapped = ZAddress::remapped(start);
+  const zaddress_unsafe addr = ZOffset::address_unsafe(start);
 
   // Reserve address space
-  if (!pd_reserve(marked0, size)) {
-    return false;
-  }
-
-  if (!pd_reserve(marked1, size)) {
-    pd_unreserve(marked0, size);
-    return false;
-  }
-
-  if (!pd_reserve(remapped, size)) {
-    pd_unreserve(marked0, size);
-    pd_unreserve(marked1, size);
+  if (!pd_reserve(addr, size)) {
     return false;
   }
 
   // Register address views with native memory tracker
-  nmt_reserve(marked0, size);
-  nmt_reserve(marked1, size);
-  nmt_reserve(remapped, size);
+  nmt_reserve(addr, size);
 
   // Make the address range free
   _manager.free(start, size);
@@ -142,8 +122,8 @@ bool ZVirtualMemoryManager::reserve_contiguous(size_t size) {
   const size_t unused = ZAddressOffsetMax - size;
   const size_t increment = MAX2(align_up(unused / 8192, ZGranuleSize), ZGranuleSize);
 
-  for (size_t start = 0; start + size <= ZAddressOffsetMax; start += increment) {
-    if (reserve_contiguous(start, size)) {
+  for (uintptr_t start = 0; start + size <= ZAddressOffsetMax; start += increment) {
+    if (reserve_contiguous(to_zoffset(start), size)) {
       // Success
       return true;
     }
@@ -154,7 +134,7 @@ bool ZVirtualMemoryManager::reserve_contiguous(size_t size) {
 }
 
 bool ZVirtualMemoryManager::reserve(size_t max_capacity) {
-  const size_t limit = MIN2(ZAddressOffsetMax, ZAddressSpaceLimit::heap_view());
+  const size_t limit = MIN2(ZAddressOffsetMax, ZAddressSpaceLimit::heap());
   const size_t size = MIN2(max_capacity * ZVirtualToPhysicalRatio, limit);
 
   size_t reserved = size;
@@ -171,8 +151,7 @@ bool ZVirtualMemoryManager::reserve(size_t max_capacity) {
                        (contiguous ? "Contiguous" : "Discontiguous"),
                        (limit == ZAddressOffsetMax ? "Unrestricted" : "Restricted"),
                        (reserved == size ? "Complete" : "Degraded"));
-  log_info_p(gc, init)("Address Space Size: " SIZE_FORMAT "M x " SIZE_FORMAT " = " SIZE_FORMAT "M",
-                       reserved / M, ZHeapViews, (reserved * ZHeapViews) / M);
+  log_info_p(gc, init)("Address Space Size: " SIZE_FORMAT "M", reserved / M);
 
   // Record reserved
   _reserved = reserved;
@@ -180,9 +159,9 @@ bool ZVirtualMemoryManager::reserve(size_t max_capacity) {
   return reserved >= max_capacity;
 }
 
-void ZVirtualMemoryManager::nmt_reserve(uintptr_t start, size_t size) {
-  MemTracker::record_virtual_memory_reserve((void*)start, size, CALLER_PC);
-  MemTracker::record_virtual_memory_type((void*)start, mtJavaHeap);
+void ZVirtualMemoryManager::nmt_reserve(zaddress_unsafe start, size_t size) {
+  MemTracker::record_virtual_memory_reserve((void*)untype(start), size, CALLER_PC);
+  MemTracker::record_virtual_memory_type((void*)untype(start), mtJavaHeap);
 }
 
 bool ZVirtualMemoryManager::is_initialized() const {
@@ -190,7 +169,7 @@ bool ZVirtualMemoryManager::is_initialized() const {
 }
 
 ZVirtualMemory ZVirtualMemoryManager::alloc(size_t size, bool force_low_address) {
-  uintptr_t start;
+  zoffset start;
 
   // Small pages are allocated at low addresses, while medium/large pages
   // are allocated at high addresses (unless forced to be at a low address).
diff --git a/src/hotspot/share/gc/z/zVirtualMemory.hpp b/src/hotspot/share/gc/z/zVirtualMemory.hpp
index a7cde86b0d1..153cad81be4 100644
--- a/src/hotspot/share/gc/z/zVirtualMemory.hpp
+++ b/src/hotspot/share/gc/z/zVirtualMemory.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,16 +30,16 @@ class ZVirtualMemory {
   friend class VMStructs;
 
 private:
-  uintptr_t _start;
-  uintptr_t _end;
+  zoffset     _start;
+  zoffset_end _end;
 
 public:
   ZVirtualMemory();
-  ZVirtualMemory(uintptr_t start, size_t size);
+  ZVirtualMemory(zoffset start, size_t size);
 
   bool is_null() const;
-  uintptr_t start() const;
-  uintptr_t end() const;
+  zoffset start() const;
+  zoffset_end end() const;
   size_t size() const;
 
   ZVirtualMemory split(size_t size);
@@ -48,22 +48,22 @@ public:
 class ZVirtualMemoryManager {
 private:
   ZMemoryManager _manager;
-  uintptr_t      _reserved;
+  size_t         _reserved;
   bool           _initialized;
 
   // Platform specific implementation
   void pd_initialize_before_reserve();
   void pd_initialize_after_reserve();
-  bool pd_reserve(uintptr_t addr, size_t size);
-  void pd_unreserve(uintptr_t addr, size_t size);
+  bool pd_reserve(zaddress_unsafe addr, size_t size);
+  void pd_unreserve(zaddress_unsafe addr, size_t size);
 
-  bool reserve_contiguous(uintptr_t start, size_t size);
+  bool reserve_contiguous(zoffset start, size_t size);
   bool reserve_contiguous(size_t size);
-  size_t reserve_discontiguous(uintptr_t start, size_t size, size_t min_range);
+  size_t reserve_discontiguous(zoffset start, size_t size, size_t min_range);
   size_t reserve_discontiguous(size_t size);
   bool reserve(size_t max_capacity);
 
-  void nmt_reserve(uintptr_t start, size_t size);
+  void nmt_reserve(zaddress_unsafe start, size_t size);
 
 public:
   ZVirtualMemoryManager(size_t max_capacity);
@@ -71,7 +71,7 @@ public:
   bool is_initialized() const;
 
   size_t reserved() const;
-  uintptr_t lowest_available_address() const;
+  zoffset lowest_available_address() const;
 
   ZVirtualMemory alloc(size_t size, bool force_low_address);
   void free(const ZVirtualMemory& vmem);
diff --git a/src/hotspot/share/gc/z/zVirtualMemory.inline.hpp b/src/hotspot/share/gc/z/zVirtualMemory.inline.hpp
index 20071fa4490..83aba75f675 100644
--- a/src/hotspot/share/gc/z/zVirtualMemory.inline.hpp
+++ b/src/hotspot/share/gc/z/zVirtualMemory.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,22 +29,22 @@
 #include "gc/z/zMemory.inline.hpp"
 
 inline ZVirtualMemory::ZVirtualMemory() :
-    _start(UINTPTR_MAX),
-    _end(UINTPTR_MAX) {}
+    _start(zoffset(UINTPTR_MAX)),
+    _end(zoffset_end(UINTPTR_MAX)) {}
 
-inline ZVirtualMemory::ZVirtualMemory(uintptr_t start, size_t size) :
+inline ZVirtualMemory::ZVirtualMemory(zoffset start, size_t size) :
     _start(start),
-    _end(start + size) {}
+    _end(to_zoffset_end(start, size)) {}
 
 inline bool ZVirtualMemory::is_null() const {
-  return _start == UINTPTR_MAX;
+  return _start == zoffset(UINTPTR_MAX);
 }
 
-inline uintptr_t ZVirtualMemory::start() const {
+inline zoffset ZVirtualMemory::start() const {
   return _start;
 }
 
-inline uintptr_t ZVirtualMemory::end() const {
+inline zoffset_end ZVirtualMemory::end() const {
   return _end;
 }
 
@@ -61,7 +61,7 @@ inline size_t ZVirtualMemoryManager::reserved() const {
   return _reserved;
 }
 
-inline uintptr_t ZVirtualMemoryManager::lowest_available_address() const {
+inline zoffset ZVirtualMemoryManager::lowest_available_address() const {
   return _manager.peek_low_address();
 }
 
diff --git a/src/hotspot/share/gc/z/zWeakRootsProcessor.cpp b/src/hotspot/share/gc/z/zWeakRootsProcessor.cpp
index 89163c7687f..5b01ad30ba6 100644
--- a/src/hotspot/share/gc/z/zWeakRootsProcessor.cpp
+++ b/src/hotspot/share/gc/z/zWeakRootsProcessor.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,28 +22,23 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/shared/suspendibleThreadSet.hpp"
+#include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zBarrier.inline.hpp"
+#include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zRootsIterator.hpp"
 #include "gc/z/zTask.hpp"
 #include "gc/z/zWeakRootsProcessor.hpp"
 #include "gc/z/zWorkers.hpp"
+#include "memory/iterator.hpp"
+#include "runtime/atomic.hpp"
+#include "utilities/debug.hpp"
 
 class ZPhantomCleanOopClosure : public OopClosure {
 public:
   virtual void do_oop(oop* p) {
-    // Read the oop once, to make sure the liveness check
-    // and the later clearing uses the same value.
-    const oop obj = Atomic::load(p);
-    if (ZBarrier::is_alive_barrier_on_phantom_oop(obj)) {
-      ZBarrier::keep_alive_barrier_on_phantom_oop_field(p);
-    } else {
-      // The destination could have been modified/reused, in which case
-      // we don't want to clear it. However, no one could write the same
-      // oop here again (the object would be strongly live and we would
-      // not consider clearing such oops), so therefore we don't have an
-      // ABA problem here.
-      Atomic::cmpxchg(p, obj, oop(NULL));
-    }
+    ZBarrier::clean_barrier_on_phantom_oop_field((zpointer*)p);
+    SuspendibleThreadSet::yield();
   }
 
   virtual void do_oop(narrowOop* p) {
@@ -56,20 +51,21 @@ ZWeakRootsProcessor::ZWeakRootsProcessor(ZWorkers* workers) :
 
 class ZProcessWeakRootsTask : public ZTask {
 private:
-  ZWeakRootsIterator _weak_roots;
+  ZRootsIteratorWeakColored _roots_weak_colored;
 
 public:
   ZProcessWeakRootsTask() :
       ZTask("ZProcessWeakRootsTask"),
-      _weak_roots() {}
+      _roots_weak_colored(ZGenerationIdOptional::old) {}
 
   ~ZProcessWeakRootsTask() {
-    _weak_roots.report_num_dead();
+    _roots_weak_colored.report_num_dead();
   }
 
   virtual void work() {
+    SuspendibleThreadSetJoiner sts_joiner;
     ZPhantomCleanOopClosure cl;
-    _weak_roots.apply(&cl);
+    _roots_weak_colored.apply(&cl);
   }
 };
 
diff --git a/src/hotspot/share/gc/z/zWorkers.cpp b/src/hotspot/share/gc/z/zWorkers.cpp
index 84f17f299fb..0e1b969848e 100644
--- a/src/hotspot/share/gc/z/zWorkers.cpp
+++ b/src/hotspot/share/gc/z/zWorkers.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,52 +24,38 @@
 #include "precompiled.hpp"
 #include "gc/shared/gc_globals.hpp"
 #include "gc/shared/gcLogPrecious.hpp"
+#include "gc/z/zHeap.inline.hpp"
 #include "gc/z/zLock.inline.hpp"
 #include "gc/z/zStat.hpp"
 #include "gc/z/zTask.hpp"
-#include "gc/z/zThread.hpp"
 #include "gc/z/zWorkers.hpp"
 #include "runtime/java.hpp"
 
-class ZWorkersInitializeTask : public WorkerTask {
-private:
-  const uint     _nworkers;
-  uint           _started;
-  ZConditionLock _lock;
+static const char* workers_name(ZGenerationId id) {
+  return (id == ZGenerationId::young) ? "ZWorkerYoung" : "ZWorkerOld";
+}
 
-public:
-  ZWorkersInitializeTask(uint nworkers) :
-      WorkerTask("ZWorkersInitializeTask"),
-      _nworkers(nworkers),
-      _started(0),
-      _lock() {}
+static const char* generation_name(ZGenerationId id) {
+  return (id == ZGenerationId::young) ? "Young" : "Old";
+}
 
-  virtual void work(uint worker_id) {
-    // Register as worker
-    ZThread::set_worker();
+static uint max_nworkers(ZGenerationId id) {
+  return id == ZGenerationId::young ? ZYoungGCThreads : ZOldGCThreads;
+}
 
-    // Wait for all threads to start
-    ZLocker<ZConditionLock> locker(&_lock);
-    if (++_started == _nworkers) {
-      // All threads started
-      _lock.notify_all();
-    } else {
-      while (_started != _nworkers) {
-        _lock.wait();
-      }
-    }
-  }
-};
+ZWorkers::ZWorkers(ZGenerationId id, ZStatWorkers* stats) :
+    _workers(workers_name(id),
+             max_nworkers(id)),
+    _generation_name(generation_name(id)),
+    _resize_lock(),
+    _requested_nworkers(0),
+    _is_active(false),
+    _stats(stats) {
 
-ZWorkers::ZWorkers() :
-    _workers("ZWorker",
-             UseDynamicNumberOfGCThreads ? ConcGCThreads : MAX2(ConcGCThreads, ParallelGCThreads)) {
-
-  if (UseDynamicNumberOfGCThreads) {
-    log_info_p(gc, init)("GC Workers: %u (dynamic)", _workers.max_workers());
-  } else {
-    log_info_p(gc, init)("GC Workers: %u/%u (static)", ConcGCThreads, _workers.max_workers());
-  }
+  log_info_p(gc, init)("GC Workers for %s Generation: %u (%s)",
+                       _generation_name,
+                       _workers.max_workers(),
+                       UseDynamicNumberOfGCThreads ? "dynamic" : "static");
 
   // Initialize worker threads
   _workers.initialize_workers();
@@ -77,10 +63,10 @@ ZWorkers::ZWorkers() :
   if (_workers.active_workers() != _workers.max_workers()) {
     vm_exit_during_initialization("Failed to create ZWorkers");
   }
+}
 
-  // Execute task to register threads as workers
-  ZWorkersInitializeTask task(_workers.max_workers());
-  _workers.run_task(&task);
+bool ZWorkers::is_active() const {
+  return _is_active;
 }
 
 uint ZWorkers::active_workers() const {
@@ -88,24 +74,63 @@ uint ZWorkers::active_workers() const {
 }
 
 void ZWorkers::set_active_workers(uint nworkers) {
-  log_info(gc, task)("Using %u workers", nworkers);
+  log_info(gc, task)("Using %u Workers for %s Generation", nworkers, _generation_name);
+  ZLocker<ZLock> locker(&_resize_lock);
   _workers.set_active_workers(nworkers);
 }
 
+void ZWorkers::set_active() {
+  ZLocker<ZLock> locker(&_resize_lock);
+  _is_active = true;
+  _requested_nworkers = 0;
+}
+
+void ZWorkers::set_inactive() {
+  ZLocker<ZLock> locker(&_resize_lock);
+  _is_active = false;
+}
+
 void ZWorkers::run(ZTask* task) {
-  log_debug(gc, task)("Executing Task: %s, Active Workers: %u", task->name(), active_workers());
-  ZStatWorkers::at_start();
+  log_debug(gc, task)("Executing %s using %s with %u workers", task->name(), _workers.name(), active_workers());
+
+  {
+    ZLocker<ZLock> locker(&_resize_lock);
+    _stats->at_start(active_workers());
+  }
+
   _workers.run_task(task->worker_task());
-  ZStatWorkers::at_end();
+
+  {
+    ZLocker<ZLock> locker(&_resize_lock);
+    _stats->at_end();
+  }
+}
+
+void ZWorkers::run(ZRestartableTask* task) {
+  for (;;) {
+    // Run task
+    run(static_cast<ZTask*>(task));
+
+    ZLocker<ZLock> locker(&_resize_lock);
+    if (_requested_nworkers == 0) {
+      // Task completed
+      return;
+    }
+
+    // Restart task with requested number of active workers
+    _workers.set_active_workers(_requested_nworkers);
+    task->resize_workers(active_workers());
+    _requested_nworkers = 0;
+  }
 }
 
 void ZWorkers::run_all(ZTask* task) {
-  // Save number of active workers
+  // Get and set number of active workers
   const uint prev_active_workers = _workers.active_workers();
+  _workers.set_active_workers(_workers.max_workers());
 
   // Execute task using all workers
-  _workers.set_active_workers(_workers.max_workers());
-  log_debug(gc, task)("Executing Task: %s, Active Workers: %u", task->name(), active_workers());
+  log_debug(gc, task)("Executing %s using %s with %u workers", task->name(), _workers.name(), active_workers());
   _workers.run_task(task->worker_task());
 
   // Restore number of active workers
@@ -115,3 +140,28 @@ void ZWorkers::run_all(ZTask* task) {
 void ZWorkers::threads_do(ThreadClosure* tc) const {
   _workers.threads_do(tc);
 }
+
+ZLock* ZWorkers::resizing_lock() {
+  return &_resize_lock;
+}
+
+void ZWorkers::request_resize_workers(uint nworkers) {
+  assert(nworkers != 0, "Never ask for zero workers");
+
+  ZLocker<ZLock> locker(&_resize_lock);
+
+  if (_requested_nworkers == nworkers) {
+    // Already requested
+    return;
+  }
+
+  if (_workers.active_workers() == nworkers) {
+    // Already the right amount of threads
+    return;
+  }
+
+  log_info(gc, task)("Adjusting Workers for %s Generation: %u -> %u",
+                     _generation_name, _workers.active_workers(), nworkers);
+
+  _requested_nworkers = nworkers;
+}
diff --git a/src/hotspot/share/gc/z/zWorkers.hpp b/src/hotspot/share/gc/z/zWorkers.hpp
index 3ee14ece6be..8d1f4152562 100644
--- a/src/hotspot/share/gc/z/zWorkers.hpp
+++ b/src/hotspot/share/gc/z/zWorkers.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,24 +25,45 @@
 #define SHARE_GC_Z_ZWORKERS_HPP
 
 #include "gc/shared/workerThread.hpp"
+#include "gc/z/zGenerationId.hpp"
+#include "gc/z/zLock.hpp"
+#include "gc/z/zStat.hpp"
 
 class ThreadClosure;
+class ZRestartableTask;
+class ZStatCycle;
+class ZStatWorkers;
 class ZTask;
 
 class ZWorkers {
 private:
-  WorkerThreads _workers;
+  WorkerThreads       _workers;
+  const char* const   _generation_name;
+  ZLock               _resize_lock;
+  volatile uint       _requested_nworkers;
+  bool                _is_active;
+  ZStatWorkers* const _stats;
 
 public:
-  ZWorkers();
+  ZWorkers(ZGenerationId id, ZStatWorkers* stats);
 
+  bool is_active() const;
   uint active_workers() const;
   void set_active_workers(uint nworkers);
+  void set_active();
+  void set_inactive();
 
   void run(ZTask* task);
+  void run(ZRestartableTask* task);
   void run_all(ZTask* task);
 
   void threads_do(ThreadClosure* tc) const;
+
+  // Worker resizing
+  ZLock* resizing_lock();
+  void request_resize_workers(uint nworkers);
+
+  bool should_worker_resize();
 };
 
 #endif // SHARE_GC_Z_ZWORKERS_HPP
diff --git a/src/hotspot/share/gc/z/zWorkers.inline.hpp b/src/hotspot/share/gc/z/zWorkers.inline.hpp
new file mode 100644
index 00000000000..ee1c5f476e3
--- /dev/null
+++ b/src/hotspot/share/gc/z/zWorkers.inline.hpp
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#ifndef SHARE_GC_Z_ZWORKERS_INLINE_HPP
+#define SHARE_GC_Z_ZWORKERS_INLINE_HPP
+
+#include "gc/z/zWorkers.hpp"
+
+#include "runtime/atomic.hpp"
+
+inline bool ZWorkers::should_worker_resize() {
+  return Atomic::load(&_requested_nworkers) != 0;
+}
+
+#endif // SHARE_GC_Z_ZWORKERS_INLINE_HPP
diff --git a/src/hotspot/share/gc/z/z_globals.hpp b/src/hotspot/share/gc/z/z_globals.hpp
index 4a49d775c8c..1ff63ca9c6c 100644
--- a/src/hotspot/share/gc/z/z_globals.hpp
+++ b/src/hotspot/share/gc/z/z_globals.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -24,6 +24,8 @@
 #ifndef SHARE_GC_Z_Z_GLOBALS_HPP
 #define SHARE_GC_Z_Z_GLOBALS_HPP
 
+#include "zPageAge.hpp"
+
 #define GC_Z_FLAGS(develop,                                                 \
                    develop_pd,                                              \
                    product,                                                 \
@@ -32,50 +34,41 @@
                    range,                                                   \
                    constraint)                                              \
                                                                             \
-  product(double, ZAllocationSpikeTolerance, 2.0,                           \
-          "Allocation spike tolerance factor")                              \
+  product(double, ZYoungCompactionLimit, 25.0,                              \
+          "Maximum allowed garbage in young pages")                         \
                                                                             \
-  product(double, ZFragmentationLimit, 25.0,                                \
-          "Maximum allowed heap fragmentation")                             \
+  product(double, ZCollectionIntervalMinor, -1,                             \
+          "Force Minor GC at a fixed time interval (in seconds)")           \
                                                                             \
-  product(size_t, ZMarkStackSpaceLimit, 8*G,                                \
-          "Maximum number of bytes allocated for mark stacks")              \
-          range(32*M, 1024*G)                                               \
-                                                                            \
-  product(double, ZCollectionInterval, 0,                                   \
+  product(double, ZCollectionIntervalMajor, -1,                             \
           "Force GC at a fixed time interval (in seconds)")                 \
                                                                             \
-  product(bool, ZProactive, true,                                           \
-          "Enable proactive GC cycles")                                     \
+  product(bool, ZCollectionIntervalOnly, false,                             \
+          "Only use timers for GC heuristics")                              \
                                                                             \
-  product(bool, ZUncommit, true,                                            \
-          "Uncommit unused memory")                                         \
+  product(bool, ZBufferStoreBarriers, true, DIAGNOSTIC,                     \
+          "Buffer store barriers")                                          \
                                                                             \
-  product(uintx, ZUncommitDelay, 5 * 60,                                    \
-          "Uncommit memory if it has been unused for the specified "        \
-          "amount of time (in seconds)")                                    \
+  product(uint, ZYoungGCThreads, 0, DIAGNOSTIC,                             \
+          "Number of GC threads for the young generation")                  \
                                                                             \
-  product(uint, ZStatisticsInterval, 10, DIAGNOSTIC,                        \
-          "Time between statistics print outs (in seconds)")                \
-          range(1, (uint)-1)                                                \
+  product(uint, ZOldGCThreads, 0, DIAGNOSTIC,                               \
+          "Number of GC threads for the old generation")                    \
                                                                             \
-  product(bool, ZStressRelocateInPlace, false, DIAGNOSTIC,                  \
-          "Always relocate pages in-place")                                 \
+  product(uintx, ZIndexDistributorStrategy, 0, DIAGNOSTIC,                  \
+          "Strategy used to distribute indices to parallel workers "        \
+          "0: Claim tree "                                                  \
+          "1: Simple Striped ")                                             \
                                                                             \
-  product(bool, ZVerifyViews, false, DIAGNOSTIC,                            \
-          "Verify heap view accesses")                                      \
+  product(bool, ZVerifyRemembered, trueInDebug, DIAGNOSTIC,                 \
+          "Verify remembered sets")                                         \
                                                                             \
-  product(bool, ZVerifyRoots, trueInDebug, DIAGNOSTIC,                      \
-          "Verify roots")                                                   \
+  develop(bool, ZVerifyOops, false,                                         \
+          "Verify accessed oops")                                           \
                                                                             \
-  product(bool, ZVerifyObjects, false, DIAGNOSTIC,                          \
-          "Verify objects")                                                 \
-                                                                            \
-  product(bool, ZVerifyMarking, trueInDebug, DIAGNOSTIC,                    \
-          "Verify marking stacks")                                          \
-                                                                            \
-  product(bool, ZVerifyForwarding, false, DIAGNOSTIC,                       \
-          "Verify forwarding tables")
+  product(int, ZTenuringThreshold, -1, DIAGNOSTIC,                          \
+          "Young generation tenuring threshold, -1 for dynamic computation")\
+          range(-1, static_cast<int>(ZPageAgeMax))
 
 // end of GC_Z_FLAGS
 
diff --git a/src/hotspot/share/jfr/metadata/metadata.xml b/src/hotspot/share/jfr/metadata/metadata.xml
index 4a06f44fe26..d04e0825d28 100644
--- a/src/hotspot/share/jfr/metadata/metadata.xml
+++ b/src/hotspot/share/jfr/metadata/metadata.xml
@@ -333,7 +333,7 @@
     <Field type="uint" name="numberOfRegions" label="Number of Regions" />
   </Event>
 
-  <Event name="GarbageCollection" category="Java Virtual Machine, GC, Collector" label="Garbage Collection" description="Garbage collection performed by the JVM">
+  <Event name="GarbageCollection" category="Java Virtual Machine, GC, Collector" label="Garbage Collection" description="Garbage collection performed by the JVM" thread="true">
     <Field type="uint" name="gcId" label="GC Identifier" relation="GcId" />
     <Field type="GCName" name="name" label="Name" description="The name of the Garbage Collector" />
     <Field type="GCCause" name="cause" label="Cause" description="The reason for triggering this Garbage Collection" />
@@ -539,6 +539,11 @@
     <Field type="string" name="name" label="Name" />
   </Event>
 
+  <Event name="GCPhaseConcurrentLevel2" category="Java Virtual Machine, GC, Phases" label="GC Phase Concurrent Level 2" thread="true">
+    <Field type="uint" name="gcId" label="GC Identifier" relation="GcId" />
+    <Field type="string" name="name" label="Name" />
+  </Event>
+
   <Event name="GCPhaseParallel" category="Java Virtual Machine, GC, Phases" label="GC Phase Parallel"
          startTime="true" thread="true" description="GC phases for parallel workers">
     <Field type="uint" name="gcId" label="GC Identifier" relation="GcId"/>
@@ -1071,6 +1076,15 @@
     <Field type="uint" name="newRatio" label="New Ratio" description="The size of the young generation relative to the tenured generation" />
   </Event>
 
+  <Event name="ZYoungGarbageCollection" category="Java Virtual Machine, GC, Collector" label="ZGC Young Garbage Collection" description="Extra information specific to ZGC Young Garbage Collections" thread="true">
+    <Field type="uint" name="gcId" label="GC Identifier" relation="GcId" />
+    <Field type="uint" name="tenuringThreshold" label="Tenuring Threshold" />
+  </Event>
+
+  <Event name="ZOldGarbageCollection" category="Java Virtual Machine, GC, Collector" label="ZGC Old Garbage Collection" description="Extra information specific to ZGC Old Garbage Collections" thread="true">
+    <Field type="uint" name="gcId" label="GC Identifier" relation="GcId" />
+  </Event>
+
   <Event name="ZAllocationStall" category="Java Virtual Machine, GC, Detailed" label="ZGC Allocation Stall" description="Time spent waiting for memory to become available" thread="true">
     <Field type="ZPageTypeType" name="type" label="Type" />
     <Field type="ulong" contentType="bytes" name="size" label="Size" />
@@ -1093,9 +1107,10 @@
 
   <Event name="ZRelocationSetGroup" category="Java Virtual Machine, GC, Detailed" label="ZGC Relocation Set Group" thread="true">
     <Field type="ZPageTypeType" name="type" label="Type" />
-    <Field type="ulong" name="pages" label="Pages" />
+    <Field type="ulong" name="candidatePages" label="Candidate Pages" />
     <Field type="ulong" contentType="bytes" name="total" label="Total" />
     <Field type="ulong" contentType="bytes" name="empty" label="Empty" />
+    <Field type="ulong" name="selectedPages" label="Selected Pages" />
     <Field type="ulong" contentType="bytes" name="relocate" label="Relocate" />
   </Event>
 
diff --git a/src/hotspot/share/jvmci/jvmciCompilerToVMInit.cpp b/src/hotspot/share/jvmci/jvmciCompilerToVMInit.cpp
index c511e5051c9..1b36905fea1 100644
--- a/src/hotspot/share/jvmci/jvmciCompilerToVMInit.cpp
+++ b/src/hotspot/share/jvmci/jvmciCompilerToVMInit.cpp
@@ -33,8 +33,8 @@
 #include "gc/shared/gc_globals.hpp"
 #include "gc/shared/tlab_globals.hpp"
 #if INCLUDE_ZGC
-#include "gc/z/zBarrierSetRuntime.hpp"
-#include "gc/z/zThreadLocalData.hpp"
+#include "gc/x/xBarrierSetRuntime.hpp"
+#include "gc/x/xThreadLocalData.hpp"
 #endif
 #include "jvmci/jvmciCompilerToVM.hpp"
 #include "jvmci/jvmciEnv.hpp"
@@ -149,15 +149,15 @@ void CompilerToVM::Data::initialize(JVMCI_TRAPS) {
 
 #if INCLUDE_ZGC
   if (UseZGC) {
-    thread_address_bad_mask_offset = in_bytes(ZThreadLocalData::address_bad_mask_offset());
-    ZBarrierSetRuntime_load_barrier_on_oop_field_preloaded =                     ZBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr();
-    ZBarrierSetRuntime_load_barrier_on_weak_oop_field_preloaded =                ZBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr();
-    ZBarrierSetRuntime_load_barrier_on_phantom_oop_field_preloaded =             ZBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded_addr();
-    ZBarrierSetRuntime_weak_load_barrier_on_oop_field_preloaded =                ZBarrierSetRuntime::weak_load_barrier_on_oop_field_preloaded_addr();
-    ZBarrierSetRuntime_weak_load_barrier_on_weak_oop_field_preloaded =           ZBarrierSetRuntime::weak_load_barrier_on_weak_oop_field_preloaded_addr();
-    ZBarrierSetRuntime_weak_load_barrier_on_phantom_oop_field_preloaded =        ZBarrierSetRuntime::weak_load_barrier_on_phantom_oop_field_preloaded_addr();
-    ZBarrierSetRuntime_load_barrier_on_oop_array =                               ZBarrierSetRuntime::load_barrier_on_oop_array_addr();
-    ZBarrierSetRuntime_clone =                                                   ZBarrierSetRuntime::clone_addr();
+    thread_address_bad_mask_offset = in_bytes(XThreadLocalData::address_bad_mask_offset());
+    ZBarrierSetRuntime_load_barrier_on_oop_field_preloaded =                     XBarrierSetRuntime::load_barrier_on_oop_field_preloaded_addr();
+    ZBarrierSetRuntime_load_barrier_on_weak_oop_field_preloaded =                XBarrierSetRuntime::load_barrier_on_weak_oop_field_preloaded_addr();
+    ZBarrierSetRuntime_load_barrier_on_phantom_oop_field_preloaded =             XBarrierSetRuntime::load_barrier_on_phantom_oop_field_preloaded_addr();
+    ZBarrierSetRuntime_weak_load_barrier_on_oop_field_preloaded =                XBarrierSetRuntime::weak_load_barrier_on_oop_field_preloaded_addr();
+    ZBarrierSetRuntime_weak_load_barrier_on_weak_oop_field_preloaded =           XBarrierSetRuntime::weak_load_barrier_on_weak_oop_field_preloaded_addr();
+    ZBarrierSetRuntime_weak_load_barrier_on_phantom_oop_field_preloaded =        XBarrierSetRuntime::weak_load_barrier_on_phantom_oop_field_preloaded_addr();
+    ZBarrierSetRuntime_load_barrier_on_oop_array =                               XBarrierSetRuntime::load_barrier_on_oop_array_addr();
+    ZBarrierSetRuntime_clone =                                                   XBarrierSetRuntime::clone_addr();
   }
 #endif
 
diff --git a/src/hotspot/share/jvmci/jvmci_globals.cpp b/src/hotspot/share/jvmci/jvmci_globals.cpp
index 99281752c1d..3817d7ef4ce 100644
--- a/src/hotspot/share/jvmci/jvmci_globals.cpp
+++ b/src/hotspot/share/jvmci/jvmci_globals.cpp
@@ -211,7 +211,7 @@ bool JVMCIGlobals::enable_jvmci_product_mode(JVMFlagOrigin origin) {
 }
 
 bool JVMCIGlobals::gc_supports_jvmci() {
-  return UseSerialGC || UseParallelGC || UseG1GC || UseZGC;
+  return UseSerialGC || UseParallelGC || UseG1GC || (UseZGC && !ZGenerational);
 }
 
 void JVMCIGlobals::check_jvmci_supported_gc() {
diff --git a/src/hotspot/share/logging/logPrefix.hpp b/src/hotspot/share/logging/logPrefix.hpp
index 13650157303..49f7b7991c3 100644
--- a/src/hotspot/share/logging/logPrefix.hpp
+++ b/src/hotspot/share/logging/logPrefix.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -77,6 +77,7 @@ DEBUG_ONLY(size_t Test_log_prefix_prefixer(char* buf, size_t len);)
   LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, phases, verify, start)) \
   LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, plab)) \
   LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, promotion)) \
+  LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, page)) \
   LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, region)) \
   LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, remset)) \
   LOG_PREFIX(GCId::print_prefix, LOG_TAGS(gc, remset, tracking)) \
diff --git a/src/hotspot/share/logging/logTag.hpp b/src/hotspot/share/logging/logTag.hpp
index edf33afa43d..5a524213b4e 100644
--- a/src/hotspot/share/logging/logTag.hpp
+++ b/src/hotspot/share/logging/logTag.hpp
@@ -138,6 +138,7 @@ class outputStream;
   LOG_TAG(oopstorage) \
   LOG_TAG(os) \
   LOG_TAG(owner) \
+  LOG_TAG(page) \
   LOG_TAG(pagesize) \
   LOG_TAG(parser) \
   LOG_TAG(patch) \
diff --git a/src/hotspot/share/memory/iterator.hpp b/src/hotspot/share/memory/iterator.hpp
index 5d5c494ef11..b936cac4abd 100644
--- a/src/hotspot/share/memory/iterator.hpp
+++ b/src/hotspot/share/memory/iterator.hpp
@@ -204,12 +204,16 @@ class ObjectClosure : public Closure {
   virtual void do_object(oop obj) = 0;
 };
 
-
 class BoolObjectClosure : public Closure {
  public:
   virtual bool do_object_b(oop obj) = 0;
 };
 
+class OopFieldClosure {
+public:
+  virtual void do_field(oop base, oop* p) = 0;
+};
+
 class AlwaysTrueClosure: public BoolObjectClosure {
  public:
   bool do_object_b(oop p) { return true; }
diff --git a/src/hotspot/share/memory/universe.cpp b/src/hotspot/share/memory/universe.cpp
index a7cf043aa7e..90511dee2bc 100644
--- a/src/hotspot/share/memory/universe.cpp
+++ b/src/hotspot/share/memory/universe.cpp
@@ -1216,6 +1216,11 @@ void Universe::calculate_verify_data(HeapWord* low_boundary, HeapWord* high_boun
   _verify_oop_bits = bits;
 }
 
+void Universe::set_verify_data(uintptr_t mask, uintptr_t bits) {
+  _verify_oop_mask = mask;
+  _verify_oop_bits = bits;
+}
+
 // Oop verification (see MacroAssembler::verify_oop)
 
 uintptr_t Universe::verify_oop_mask() {
diff --git a/src/hotspot/share/memory/universe.hpp b/src/hotspot/share/memory/universe.hpp
index a5eafb1076a..9cbb7ecd3f5 100644
--- a/src/hotspot/share/memory/universe.hpp
+++ b/src/hotspot/share/memory/universe.hpp
@@ -208,6 +208,7 @@ class Universe: AllStatic {
 
  public:
   static void calculate_verify_data(HeapWord* low_boundary, HeapWord* high_boundary) PRODUCT_RETURN;
+  static void set_verify_data(uintptr_t mask, uintptr_t bits) PRODUCT_RETURN;
 
   // Known classes in the VM
   static Klass* boolArrayKlassObj()                 { return typeArrayKlassObj(T_BOOLEAN); }
diff --git a/src/hotspot/share/oops/oop.cpp b/src/hotspot/share/oops/oop.cpp
index eb5624e402b..a0b3de53649 100644
--- a/src/hotspot/share/oops/oop.cpp
+++ b/src/hotspot/share/oops/oop.cpp
@@ -190,7 +190,8 @@ void* oopDesc::load_oop_raw(oop obj, int offset) {
 
 oop oopDesc::obj_field_acquire(int offset) const                      { return HeapAccess<MO_ACQUIRE>::oop_load_at(as_oop(), offset); }
 
-void oopDesc::obj_field_put_raw(int offset, oop value)                { RawAccess<>::oop_store_at(as_oop(), offset, value); }
+void oopDesc::obj_field_put_raw(int offset, oop value)                { assert(!(UseZGC && ZGenerational), "Generational ZGC must use store barriers");
+                                                                        RawAccess<>::oop_store_at(as_oop(), offset, value); }
 void oopDesc::release_obj_field_put(int offset, oop value)            { HeapAccess<MO_RELEASE>::oop_store_at(as_oop(), offset, value); }
 void oopDesc::obj_field_put_volatile(int offset, oop value)           { HeapAccess<MO_SEQ_CST>::oop_store_at(as_oop(), offset, value); }
 
diff --git a/src/hotspot/share/oops/oop.hpp b/src/hotspot/share/oops/oop.hpp
index a7be060b007..892f977edf5 100644
--- a/src/hotspot/share/oops/oop.hpp
+++ b/src/hotspot/share/oops/oop.hpp
@@ -73,6 +73,7 @@ class oopDesc {
 
   inline void set_mark(markWord m);
   static inline void set_mark(HeapWord* mem, markWord m);
+  static inline void release_set_mark(HeapWord* mem, markWord m);
 
   inline void release_set_mark(markWord m);
   inline markWord cas_set_mark(markWord new_mark, markWord old_mark);
diff --git a/src/hotspot/share/oops/oop.inline.hpp b/src/hotspot/share/oops/oop.inline.hpp
index 4988846f172..ce9d37aad36 100644
--- a/src/hotspot/share/oops/oop.inline.hpp
+++ b/src/hotspot/share/oops/oop.inline.hpp
@@ -66,6 +66,10 @@ void oopDesc::set_mark(HeapWord* mem, markWord m) {
   *(markWord*)(((char*)mem) + mark_offset_in_bytes()) = m;
 }
 
+void oopDesc::release_set_mark(HeapWord* mem, markWord m) {
+  Atomic::release_store((markWord*)(((char*)mem) + mark_offset_in_bytes()), m);
+}
+
 void oopDesc::release_set_mark(markWord m) {
   Atomic::release_store(&_mark, m);
 }
diff --git a/src/hotspot/share/oops/oopsHierarchy.hpp b/src/hotspot/share/oops/oopsHierarchy.hpp
index 0d52c523b8e..060c1b97d10 100644
--- a/src/hotspot/share/oops/oopsHierarchy.hpp
+++ b/src/hotspot/share/oops/oopsHierarchy.hpp
@@ -169,6 +169,10 @@ template <typename T> inline T cast_from_oop(oop o) {
   return (T)(CHECK_UNHANDLED_OOPS_ONLY((oopDesc*))o);
 }
 
+inline intptr_t p2i(narrowOop o) {
+  return static_cast<intptr_t>(o);
+}
+
 // The metadata hierarchy is separate from the oop hierarchy
 
 //      class MetaspaceObj
diff --git a/src/hotspot/share/oops/stackChunkOop.inline.hpp b/src/hotspot/share/oops/stackChunkOop.inline.hpp
index 6e59da94076..1d5bb7ffa81 100644
--- a/src/hotspot/share/oops/stackChunkOop.inline.hpp
+++ b/src/hotspot/share/oops/stackChunkOop.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,12 +30,14 @@
 #include "gc/shared/collectedHeap.hpp"
 #include "gc/shared/barrierSet.hpp"
 #include "gc/shared/barrierSetStackChunk.hpp"
+#include "gc/shared/gc_globals.hpp"
 #include "memory/memRegion.hpp"
 #include "memory/universe.hpp"
 #include "oops/access.inline.hpp"
 #include "oops/instanceStackChunkKlass.inline.hpp"
 #include "runtime/continuationJavaClasses.inline.hpp"
 #include "runtime/frame.inline.hpp"
+#include "runtime/globals.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/registerMap.hpp"
 #include "runtime/smallRegisterMap.inline.hpp"
@@ -86,17 +88,19 @@ inline void stackChunkOopDesc::set_max_thawing_size(int value)  {
   jdk_internal_vm_StackChunk::set_maxThawingSize(this, (jint)value);
 }
 
-inline oop stackChunkOopDesc::cont() const                { return UseCompressedOops ? cont<narrowOop>() : cont<oop>(); /* jdk_internal_vm_StackChunk::cont(as_oop()); */ }
-template<typename P>
 inline oop stackChunkOopDesc::cont() const                {
-  // The state of the cont oop is used by ZCollectedHeap::requires_barriers,
-  // to determine the age of the stackChunkOopDesc. For that to work, it is
-  // only the GC that is allowed to perform a load barrier on the oop.
-  // This function is used by non-GC code and therfore create a stack-local
-  // copy on the oop and perform the load barrier on that copy instead.
-  oop obj = jdk_internal_vm_StackChunk::cont_raw<P>(as_oop());
-  obj = (oop)NativeAccess<>::oop_load(&obj);
-  return obj;
+  if (UseZGC && !ZGenerational) {
+    assert(!UseCompressedOops, "Non-generational ZGC does not support compressed oops");
+    // The state of the cont oop is used by XCollectedHeap::requires_barriers,
+    // to determine the age of the stackChunkOopDesc. For that to work, it is
+    // only the GC that is allowed to perform a load barrier on the oop.
+    // This function is used by non-GC code and therfore create a stack-local
+    // copy on the oop and perform the load barrier on that copy instead.
+    oop obj = jdk_internal_vm_StackChunk::cont_raw<oop>(as_oop());
+    obj = (oop)NativeAccess<>::oop_load(&obj);
+    return obj;
+  }
+  return jdk_internal_vm_StackChunk::cont(as_oop());
 }
 inline void stackChunkOopDesc::set_cont(oop value)        { jdk_internal_vm_StackChunk::set_cont(this, value); }
 template<typename P>
diff --git a/src/hotspot/share/opto/output.hpp b/src/hotspot/share/opto/output.hpp
index e764e83df10..77c49b33bfe 100644
--- a/src/hotspot/share/opto/output.hpp
+++ b/src/hotspot/share/opto/output.hpp
@@ -188,6 +188,8 @@ public:
   void       set_in_scratch_emit_size(bool x)   {        _in_scratch_emit_size = x; }
   bool           in_scratch_emit_size() const   { return _in_scratch_emit_size;     }
 
+  BufferSizingData* buffer_sizing_data()        { return &_buf_sizes; }
+
   enum ScratchBufferBlob {
     MAX_inst_size       = 2048,
     MAX_locs_size       = 128, // number of relocInfo elements
diff --git a/src/hotspot/share/prims/whitebox.cpp b/src/hotspot/share/prims/whitebox.cpp
index 7891f111744..7db95c0f980 100644
--- a/src/hotspot/share/prims/whitebox.cpp
+++ b/src/hotspot/share/prims/whitebox.cpp
@@ -109,6 +109,9 @@
 #if INCLUDE_PARALLELGC
 #include "gc/parallel/parallelScavengeHeap.inline.hpp"
 #endif // INCLUDE_PARALLELGC
+#if INCLUDE_ZGC
+#include "gc/z/zAddress.inline.hpp"
+#endif // INCLUDE_ZGC
 #if INCLUDE_JVMCI
 #include "jvmci/jvmciEnv.hpp"
 #include "jvmci/jvmciRuntime.hpp"
@@ -397,7 +400,11 @@ WB_ENTRY(jboolean, WB_isObjectInOldGen(JNIEnv* env, jobject o, jobject obj))
 #endif
 #if INCLUDE_ZGC
   if (UseZGC) {
-    return Universe::heap()->is_in(p);
+    if (ZGenerational) {
+      return ZHeap::heap()->is_old(to_zaddress(p));
+    } else {
+      return Universe::heap()->is_in(p);
+    }
   }
 #endif
 #if INCLUDE_SHENANDOAHGC
diff --git a/src/hotspot/share/runtime/continuationFreezeThaw.cpp b/src/hotspot/share/runtime/continuationFreezeThaw.cpp
index 09505e4f210..c8dbcbc5e83 100644
--- a/src/hotspot/share/runtime/continuationFreezeThaw.cpp
+++ b/src/hotspot/share/runtime/continuationFreezeThaw.cpp
@@ -66,6 +66,9 @@
 #include "utilities/debug.hpp"
 #include "utilities/exceptions.hpp"
 #include "utilities/macros.hpp"
+#if INCLUDE_ZGC
+#include "gc/z/zStackChunkGCData.inline.hpp"
+#endif
 
 #include <type_traits>
 
@@ -1390,14 +1393,16 @@ stackChunkOop Freeze<ConfigT>::allocate_chunk(size_t stack_size) {
   chunk->set_cont_access<IS_DEST_UNINITIALIZED>(_cont.continuation());
 
 #if INCLUDE_ZGC
- if (UseZGC) {
+  if (UseZGC) {
+    if (ZGenerational) {
+      ZStackChunkGCData::initialize(chunk);
+    }
     assert(!chunk->requires_barriers(), "ZGC always allocates in the young generation");
     _barriers = false;
   } else
 #endif
 #if INCLUDE_SHENANDOAHGC
-if (UseShenandoahGC) {
-
+  if (UseShenandoahGC) {
     _barriers = chunk->requires_barriers();
   } else
 #endif
diff --git a/src/hotspot/share/runtime/continuationJavaClasses.hpp b/src/hotspot/share/runtime/continuationJavaClasses.hpp
index d8634161a4b..54d10d04154 100644
--- a/src/hotspot/share/runtime/continuationJavaClasses.hpp
+++ b/src/hotspot/share/runtime/continuationJavaClasses.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -124,6 +124,7 @@ class jdk_internal_vm_StackChunk: AllStatic {
   static inline void set_maxThawingSize(oop chunk, int value);
 
   // cont oop's processing is essential for the chunk's GC protocol
+  static inline oop cont(oop chunk);
   template<typename P>
   static inline oop cont_raw(oop chunk);
   static inline void set_cont(oop chunk, oop value);
diff --git a/src/hotspot/share/runtime/continuationJavaClasses.inline.hpp b/src/hotspot/share/runtime/continuationJavaClasses.inline.hpp
index 5bba4016033..987f367303d 100644
--- a/src/hotspot/share/runtime/continuationJavaClasses.inline.hpp
+++ b/src/hotspot/share/runtime/continuationJavaClasses.inline.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1997, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -81,6 +81,10 @@ inline void jdk_internal_vm_StackChunk::set_parent_access(oop chunk, oop value)
   chunk->obj_field_put_access<decorators>(_parent_offset, value);
 }
 
+inline oop jdk_internal_vm_StackChunk::cont(oop chunk) {
+  return chunk->obj_field(_cont_offset);
+}
+
 template<typename P>
 inline oop jdk_internal_vm_StackChunk::cont_raw(oop chunk) {
   return (oop)RawAccess<>::oop_load(chunk->field_addr<P>(_cont_offset));
diff --git a/src/hotspot/share/runtime/stackWatermark.cpp b/src/hotspot/share/runtime/stackWatermark.cpp
index 717c3b535c5..d499623defc 100644
--- a/src/hotspot/share/runtime/stackWatermark.cpp
+++ b/src/hotspot/share/runtime/stackWatermark.cpp
@@ -59,6 +59,8 @@ public:
 };
 
 void StackWatermarkFramesIterator::set_watermark(uintptr_t sp) {
+  assert(sp != 0, "Sanity check");
+
   if (!has_next()) {
     return;
   }
@@ -278,6 +280,10 @@ uintptr_t StackWatermark::last_processed() {
   return _iterator->caller();
 }
 
+uintptr_t StackWatermark::last_processed_raw() {
+  return _iterator->caller();
+}
+
 bool StackWatermark::processing_started() const {
   return processing_started(Atomic::load(&_state));
 }
diff --git a/src/hotspot/share/runtime/stackWatermark.hpp b/src/hotspot/share/runtime/stackWatermark.hpp
index 52d34ea5cfb..8d51f694a1d 100644
--- a/src/hotspot/share/runtime/stackWatermark.hpp
+++ b/src/hotspot/share/runtime/stackWatermark.hpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -137,6 +137,7 @@ public:
 
   uintptr_t watermark();
   uintptr_t last_processed();
+  uintptr_t last_processed_raw();
 
   bool processing_started() const;
   bool processing_started_acquire() const;
diff --git a/src/hotspot/share/runtime/thread.cpp b/src/hotspot/share/runtime/thread.cpp
index ef06eb29e9d..2fe528560c4 100644
--- a/src/hotspot/share/runtime/thread.cpp
+++ b/src/hotspot/share/runtime/thread.cpp
@@ -72,6 +72,7 @@ Thread::Thread() {
   set_stack_size(0);
   set_lgrp_id(-1);
   DEBUG_ONLY(clear_suspendible_thread();)
+  DEBUG_ONLY(clear_indirectly_suspendible_thread();)
 
   // allocated data structures
   set_osthread(nullptr);
diff --git a/src/hotspot/share/runtime/thread.hpp b/src/hotspot/share/runtime/thread.hpp
index b998879cab9..e813bc04c2a 100644
--- a/src/hotspot/share/runtime/thread.hpp
+++ b/src/hotspot/share/runtime/thread.hpp
@@ -205,6 +205,7 @@ class Thread: public ThreadShadow {
 
  private:
   DEBUG_ONLY(bool _suspendible_thread;)
+  DEBUG_ONLY(bool _indirectly_suspendible_thread;)
 
  public:
   // Determines if a heap allocation failure will be retried
@@ -216,15 +217,13 @@ class Thread: public ThreadShadow {
   virtual bool in_retryable_allocation() const { return false; }
 
 #ifdef ASSERT
-  void set_suspendible_thread() {
-    _suspendible_thread = true;
-  }
+  void set_suspendible_thread()   { _suspendible_thread = true; }
+  void clear_suspendible_thread() { _suspendible_thread = false; }
+  bool is_suspendible_thread()    { return _suspendible_thread; }
 
-  void clear_suspendible_thread() {
-    _suspendible_thread = false;
-  }
-
-  bool is_suspendible_thread() { return _suspendible_thread; }
+  void set_indirectly_suspendible_thread()   { _indirectly_suspendible_thread = true; }
+  void clear_indirectly_suspendible_thread() { _indirectly_suspendible_thread = false; }
+  bool is_indirectly_suspendible_thread()    { return _indirectly_suspendible_thread; }
 #endif
 
  private:
diff --git a/src/hotspot/share/runtime/vmOperation.hpp b/src/hotspot/share/runtime/vmOperation.hpp
index ace701b8320..09f850a84ed 100644
--- a/src/hotspot/share/runtime/vmOperation.hpp
+++ b/src/hotspot/share/runtime/vmOperation.hpp
@@ -60,10 +60,19 @@
   template(G1PauseRemark)                         \
   template(G1PauseCleanup)                        \
   template(G1TryInitiateConcMark)                 \
-  template(ZMarkStart)                            \
-  template(ZMarkEnd)                              \
-  template(ZRelocateStart)                        \
-  template(ZVerify)                               \
+  template(ZMarkEndOld)                           \
+  template(ZMarkEndYoung)                         \
+  template(ZMarkFlushOperation)                   \
+  template(ZMarkStartYoung)                       \
+  template(ZMarkStartYoungAndOld)                 \
+  template(ZRelocateStartOld)                     \
+  template(ZRelocateStartYoung)                   \
+  template(ZRendezvousGCThreads)                  \
+  template(ZVerifyOld)                            \
+  template(XMarkStart)                            \
+  template(XMarkEnd)                              \
+  template(XRelocateStart)                        \
+  template(XVerify)                               \
   template(HandshakeAllThreads)                   \
   template(PopulateDumpSharedSpace)               \
   template(JNIFunctionTableCopier)                \
diff --git a/src/hotspot/share/services/heapDumper.cpp b/src/hotspot/share/services/heapDumper.cpp
index a2c0636e56f..1d947cff1d0 100644
--- a/src/hotspot/share/services/heapDumper.cpp
+++ b/src/hotspot/share/services/heapDumper.cpp
@@ -1958,6 +1958,7 @@ class VM_HeapDumper : public VM_GC_Operation, public WorkerTask {
   }
 
   VMOp_Type type() const { return VMOp_HeapDumper; }
+  virtual bool doit_prologue();
   void doit();
   void work(uint worker_id);
 };
@@ -2135,6 +2136,17 @@ void VM_HeapDumper::do_threads() {
   }
 }
 
+bool VM_HeapDumper::doit_prologue() {
+  if (_gc_before_heap_dump && UseZGC) {
+    // ZGC cannot perform a synchronous GC cycle from within the VM thread.
+    // So ZCollectedHeap::collect_as_vm_thread() is a noop. To respect the
+    // _gc_before_heap_dump flag a synchronous GC cycle is performed from
+    // the caller thread in the prologue.
+    Universe::heap()->collect(GCCause::_heap_dump);
+  }
+  return VM_GC_Operation::doit_prologue();
+}
+
 
 // The VM operation that dumps the heap. The dump consists of the following
 // records:
diff --git a/src/hotspot/share/utilities/events.cpp b/src/hotspot/share/utilities/events.cpp
index 4f7c0231863..c47c466372e 100644
--- a/src/hotspot/share/utilities/events.cpp
+++ b/src/hotspot/share/utilities/events.cpp
@@ -36,6 +36,7 @@
 EventLog* Events::_logs = nullptr;
 StringEventLog* Events::_messages = nullptr;
 StringEventLog* Events::_vm_operations = nullptr;
+StringEventLog* Events::_zgc_phase_switch = nullptr;
 ExceptionsEventLog* Events::_exceptions = nullptr;
 StringEventLog* Events::_redefinitions = nullptr;
 UnloadingEventLog* Events::_class_unloading = nullptr;
@@ -95,6 +96,7 @@ void Events::init() {
   if (LogEvents) {
     _messages = new StringEventLog("Events", "events");
     _vm_operations = new StringEventLog("VM Operations", "vmops");
+    _zgc_phase_switch = new StringEventLog("ZGC Phase Switch", "zgcps");
     _exceptions = new ExceptionsEventLog("Internal exceptions", "exc");
     _redefinitions = new StringEventLog("Classes redefined", "redef");
     _class_unloading = new UnloadingEventLog("Classes unloaded", "unload");
diff --git a/src/hotspot/share/utilities/events.hpp b/src/hotspot/share/utilities/events.hpp
index 4494c4e4fcb..b400fd707fa 100644
--- a/src/hotspot/share/utilities/events.hpp
+++ b/src/hotspot/share/utilities/events.hpp
@@ -223,6 +223,9 @@ class Events : AllStatic {
   // A log for VM Operations
   static StringEventLog* _vm_operations;
 
+    // A log for ZGC phase switches
+  static StringEventLog* _zgc_phase_switch;
+
   // A log for internal exception related messages, like internal
   // throws and implicit exceptions.
   static ExceptionsEventLog* _exceptions;
@@ -258,6 +261,8 @@ class Events : AllStatic {
 
   static void log_vm_operation(Thread* thread, const char* format, ...) ATTRIBUTE_PRINTF(2, 3);
 
+  static void log_zgc_phase_switch(const char* format, ...) ATTRIBUTE_PRINTF(1, 2);
+
   // Log exception related message
   static void log_exception(Thread* thread, const char* format, ...) ATTRIBUTE_PRINTF(2, 3);
   static void log_exception(Thread* thread, Handle h_exception, const char* message, const char* file, int line);
@@ -294,6 +299,15 @@ inline void Events::log_vm_operation(Thread* thread, const char* format, ...) {
   }
 }
 
+inline void Events::log_zgc_phase_switch(const char* format, ...) {
+  if (LogEvents && _zgc_phase_switch != nullptr) {
+    va_list ap;
+    va_start(ap, format);
+    _zgc_phase_switch->logv(nullptr /* thread */, format, ap);
+    va_end(ap);
+  }
+}
+
 inline void Events::log_exception(Thread* thread, const char* format, ...) {
   if (LogEvents && _exceptions != nullptr) {
     va_list ap;
@@ -468,6 +482,7 @@ class EventMarkBase : public StackObj {
   void log_start(const char* format, va_list argp) ATTRIBUTE_PRINTF(2, 0);
   void log_end();
 
+ public:
   EventMarkBase(EventLogFunction log_function);
 };
 
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/HSDB.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/HSDB.java
index dbe89380bb2..9cc51934d08 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/HSDB.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/HSDB.java
@@ -38,6 +38,7 @@ import sun.jvm.hotspot.gc.parallel.*;
 import sun.jvm.hotspot.gc.shared.*;
 import sun.jvm.hotspot.gc.shenandoah.*;
 import sun.jvm.hotspot.gc.g1.*;
+import sun.jvm.hotspot.gc.x.*;
 import sun.jvm.hotspot.gc.z.*;
 import sun.jvm.hotspot.interpreter.*;
 import sun.jvm.hotspot.oops.*;
@@ -1124,6 +1125,10 @@ public class HSDB implements ObjectHistogramPanel.Listener, SAListener {
                           ShenandoahHeap heap = (ShenandoahHeap) collHeap;
                           anno = "ShenandoahHeap ";
                           bad = false;
+                        } else if (collHeap instanceof XCollectedHeap) {
+                          XCollectedHeap heap = (XCollectedHeap) collHeap;
+                          anno = "ZHeap ";
+                          bad = false;
                         } else if (collHeap instanceof ZCollectedHeap) {
                           ZCollectedHeap heap = (ZCollectedHeap) collHeap;
                           anno = "ZHeap ";
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAddress.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XAddress.java
similarity index 85%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAddress.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XAddress.java
index 52d9555e4ea..fbd15110890 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAddress.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XAddress.java
@@ -22,12 +22,12 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import sun.jvm.hotspot.debugger.Address;
 import sun.jvm.hotspot.runtime.VM;
 
-class ZAddress {
+class XAddress {
     static long as_long(Address value) {
         if (value == null) {
             return 0;
@@ -40,7 +40,7 @@ class ZAddress {
     }
 
     static boolean is_weak_bad(Address value) {
-        return (as_long(value) & ZGlobals.ZAddressWeakBadMask()) != 0L;
+        return (as_long(value) & XGlobals.XAddressWeakBadMask()) != 0L;
     }
 
     static boolean is_weak_good(Address value) {
@@ -52,11 +52,11 @@ class ZAddress {
     }
 
     static long offset(Address address) {
-        return as_long(address) & ZGlobals.ZAddressOffsetMask();
+        return as_long(address) & XGlobals.XAddressOffsetMask();
     }
 
     static Address good(Address value) {
-        return VM.getVM().getDebugger().newAddress(offset(value) | ZGlobals.ZAddressGoodMask());
+        return VM.getVM().getDebugger().newAddress(offset(value) | XGlobals.XAddressGoodMask());
     }
 
     static Address good_or_null(Address value) {
@@ -69,9 +69,9 @@ class ZAddress {
 
     static boolean isIn(Address addr) {
         long value = as_long(addr);
-        if (!isPowerOf2(value & ~ZGlobals.ZAddressOffsetMask())) {
+        if (!isPowerOf2(value & ~XGlobals.XAddressOffsetMask())) {
             return false;
         }
-        return (value & (ZGlobals.ZAddressMetadataMask() & ~ZGlobals.ZAddressMetadataFinalizable())) != 0L;
+        return (value & (XGlobals.XAddressMetadataMask() & ~XGlobals.XAddressMetadataFinalizable())) != 0L;
     }
 }
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAttachedArrayForForwarding.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XAttachedArrayForForwarding.java
similarity index 79%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAttachedArrayForForwarding.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XAttachedArrayForForwarding.java
index b9d3de4c359..0c8bb38a776 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZAttachedArrayForForwarding.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XAttachedArrayForForwarding.java
@@ -23,7 +23,7 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import sun.jvm.hotspot.debugger.Address;
 import sun.jvm.hotspot.runtime.VM;
@@ -33,7 +33,7 @@ import sun.jvm.hotspot.types.CIntegerField;
 import sun.jvm.hotspot.types.Type;
 import sun.jvm.hotspot.types.TypeDataBase;
 
-public class ZAttachedArrayForForwarding extends VMObject {
+public class XAttachedArrayForForwarding extends VMObject {
     private static CIntegerField lengthField;
 
     static {
@@ -41,12 +41,12 @@ public class ZAttachedArrayForForwarding extends VMObject {
     }
 
     private static synchronized void initialize(TypeDataBase db) {
-        Type type = db.lookupType("ZAttachedArrayForForwarding");
+        Type type = db.lookupType("XAttachedArrayForForwarding");
 
         lengthField = type.getCIntegerField("_length");
     }
 
-    public ZAttachedArrayForForwarding(Address addr) {
+    public XAttachedArrayForForwarding(Address addr) {
         super(addr);
     }
 
@@ -54,18 +54,18 @@ public class ZAttachedArrayForForwarding extends VMObject {
         return lengthField.getValue(addr);
     }
 
-    // ObjectT: ZForwarding
-    //  ArrayT: ZForwardingEntry
+    // ObjectT: XForwarding
+    //  ArrayT: XForwardingEntry
     //
     // template <typename ObjectT, typename ArrayT>
-    // inline size_t ZAttachedArray<ObjectT, ArrayT>::object_size()
+    // inline size_t XAttachedArray<ObjectT, ArrayT>::object_size()
     private long objectSize() {
-        return ZUtils.alignUp(ZForwarding.getSize(), ZForwardingEntry.getSize());
+        return XUtils.alignUp(XForwarding.getSize(), XForwardingEntry.getSize());
     }
 
     // ArrayT* operator()(const ObjectT* obj) const
-    public ZForwardingEntry get(ZForwarding obj) {
+    public XForwardingEntry get(XForwarding obj) {
         Address o = obj.getAddress().addOffsetTo(objectSize());
-        return VMObjectFactory.newObject(ZForwardingEntry.class, o);
+        return VMObjectFactory.newObject(XForwardingEntry.class, o);
     }
 }
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZBarrier.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XBarrier.java
similarity index 85%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZBarrier.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XBarrier.java
index 8d5af488528..54f9323a4e6 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZBarrier.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XBarrier.java
@@ -22,30 +22,30 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import sun.jvm.hotspot.debugger.Address;
 import sun.jvm.hotspot.runtime.VM;
 
-class ZBarrier {
+class XBarrier {
     private static boolean is_weak_good_or_null_fast_path(Address addr) {
-        return ZAddress.is_weak_good_or_null(addr);
+        return XAddress.is_weak_good_or_null(addr);
     }
 
     private static Address weak_load_barrier_on_oop_slow_path(Address addr) {
-        return ZAddress.is_weak_good(addr) ? ZAddress.good(addr) : relocate_or_remap(addr);
+        return XAddress.is_weak_good(addr) ? XAddress.good(addr) : relocate_or_remap(addr);
     }
 
     private static boolean during_relocate() {
-        return ZGlobals.ZGlobalPhase() == ZGlobals.ZPhaseRelocate;
+        return XGlobals.XGlobalPhase() == XGlobals.XPhaseRelocate;
     }
 
     private static Address relocate(Address addr) {
         return zheap().relocate_object(addr);
     }
 
-    private static ZHeap zheap() {
-        ZCollectedHeap zCollectedHeap = (ZCollectedHeap)VM.getVM().getUniverse().heap();
+    private static XHeap zheap() {
+        XCollectedHeap zCollectedHeap = (XCollectedHeap)VM.getVM().getUniverse().heap();
         return zCollectedHeap.heap();
     }
 
@@ -62,7 +62,7 @@ class ZBarrier {
         if (is_weak_good_or_null_fast_path(o)) {
             // Return the good address instead of the weak good address
             // to ensure that the currently active heap view is used.
-            return ZAddress.good_or_null(o);
+            return XAddress.good_or_null(o);
         }
 
         // Slow path
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XCollectedHeap.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XCollectedHeap.java
new file mode 100644
index 00000000000..5455a841fab
--- /dev/null
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XCollectedHeap.java
@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+package sun.jvm.hotspot.gc.x;
+
+import java.io.PrintStream;
+import java.util.Iterator;
+
+import sun.jvm.hotspot.debugger.Address;
+import sun.jvm.hotspot.debugger.OopHandle;
+import sun.jvm.hotspot.gc.shared.CollectedHeap;
+import sun.jvm.hotspot.gc.shared.CollectedHeapName;
+import sun.jvm.hotspot.gc.shared.LiveRegionsClosure;
+import sun.jvm.hotspot.runtime.VM;
+import sun.jvm.hotspot.runtime.VMObjectFactory;
+import sun.jvm.hotspot.types.Type;
+import sun.jvm.hotspot.types.TypeDataBase;
+import sun.jvm.hotspot.utilities.BitMapInterface;
+
+// Mirror class for XCollectedHeap.
+
+public class XCollectedHeap extends CollectedHeap {
+    private static long zHeapFieldOffset;
+
+    static {
+        VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase()));
+    }
+
+    private static synchronized void initialize(TypeDataBase db) {
+        Type type = db.lookupType("XCollectedHeap");
+
+        zHeapFieldOffset = type.getAddressField("_heap").getOffset();
+    }
+
+    public XHeap heap() {
+        Address heapAddr = addr.addOffsetTo(zHeapFieldOffset);
+        return VMObjectFactory.newObject(XHeap.class, heapAddr);
+    }
+
+    @Override
+    public CollectedHeapName kind() {
+        return CollectedHeapName.Z;
+    }
+
+    @Override
+    public void printOn(PrintStream tty) {
+        heap().printOn(tty);
+    }
+
+    public XCollectedHeap(Address addr) {
+        super(addr);
+    }
+
+    @Override
+    public long capacity() {
+        return heap().capacity();
+    }
+
+    @Override
+    public long used() {
+        return heap().used();
+    }
+
+    @Override
+    public boolean isInReserved(Address a) {
+        return heap().isIn(a);
+    }
+
+    private OopHandle oop_load_barrier(Address oopAddress) {
+        oopAddress = XBarrier.weak_barrier(oopAddress);
+        if (oopAddress == null) {
+            return null;
+        }
+
+        return oopAddress.addOffsetToAsOopHandle(0);
+    }
+
+    @Override
+    public OopHandle oop_load_at(OopHandle handle, long offset) {
+        assert(!VM.getVM().isCompressedOopsEnabled());
+
+        Address oopAddress = handle.getAddressAt(offset);
+
+        return oop_load_barrier(oopAddress);
+    }
+
+    // addr can be either in heap or in native
+    @Override
+    public OopHandle oop_load_in_native(Address addr) {
+        Address oopAddress = addr.getAddressAt(0);
+        return oop_load_barrier(oopAddress);
+    }
+
+    public String oopAddressDescription(OopHandle handle) {
+        Address origOop = XOop.to_address(handle);
+        Address loadBarrieredOop = XBarrier.weak_barrier(origOop);
+        if (!origOop.equals(loadBarrieredOop)) {
+            return origOop + " (" + loadBarrieredOop.toString() + ")";
+        } else {
+            return handle.toString();
+        }
+    }
+
+    @Override
+    public void liveRegionsIterate(LiveRegionsClosure closure) {
+        Iterator<XPage> iter = heap().pageTable().activePagesIterator();
+        while (iter.hasNext()) {
+            XPage page = iter.next();
+            closure.doLiveRegions(page);
+        }
+    }
+
+    @Override
+    public BitMapInterface createBitMap(long size) {
+        // Ignores the size
+        return new XExternalBitMap(this);
+    }
+}
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZExternalBitMap.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XExternalBitMap.java
similarity index 82%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZExternalBitMap.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XExternalBitMap.java
index 21461129e5d..5a2f033e6a1 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZExternalBitMap.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XExternalBitMap.java
@@ -22,7 +22,7 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import java.util.HashMap;
 
@@ -31,30 +31,30 @@ import sun.jvm.hotspot.utilities.BitMap;
 import sun.jvm.hotspot.utilities.BitMapInterface;
 
 /** Discontiguous bitmap for ZGC. */
-public class ZExternalBitMap implements BitMapInterface {
-    private ZPageTable pageTable;
+public class XExternalBitMap implements BitMapInterface {
+    private XPageTable pageTable;
     private final long oopSize;
 
-    private HashMap<ZPage, BitMap> pageToBitMap = new HashMap<ZPage, BitMap>();
+    private HashMap<XPage, BitMap> pageToBitMap = new HashMap<XPage, BitMap>();
 
-    public ZExternalBitMap(ZCollectedHeap collectedHeap) {
+    public XExternalBitMap(XCollectedHeap collectedHeap) {
         pageTable = collectedHeap.heap().pageTable();
         oopSize = VM.getVM().getOopSize();
     }
 
-    private ZPage getPage(long zOffset) {
-        if (zOffset > ZGlobals.ZAddressOffsetMask()) {
+    private XPage getPage(long zOffset) {
+        if (zOffset > XGlobals.XAddressOffsetMask()) {
             throw new RuntimeException("Not a Z offset: " + zOffset);
         }
 
-        ZPage page = pageTable.get(ZUtils.longToAddress(zOffset));
+        XPage page = pageTable.get(XUtils.longToAddress(zOffset));
         if (page == null) {
             throw new RuntimeException("Address not in pageTable: " + zOffset);
         }
         return page;
     }
 
-    private BitMap getOrAddBitMap(ZPage page) {
+    private BitMap getOrAddBitMap(XPage page) {
         BitMap bitMap = pageToBitMap.get(page);
         if (bitMap == null) {
             long size = page.size();
@@ -72,20 +72,20 @@ public class ZExternalBitMap implements BitMapInterface {
         return bitMap;
     }
 
-    private int pageLocalBitMapIndex(ZPage page, long zOffset) {
+    private int pageLocalBitMapIndex(XPage page, long zOffset) {
         long pageLocalZOffset = zOffset - page.start();
         return (int)(pageLocalZOffset >>> page.object_alignment_shift());
     }
 
     private long convertToZOffset(long offset) {
         long addr = oopSize * offset;
-        return addr & ZGlobals.ZAddressOffsetMask();
+        return addr & XGlobals.XAddressOffsetMask();
     }
 
     @Override
     public boolean at(long offset) {
         long zOffset = convertToZOffset(offset);
-        ZPage page = getPage(zOffset);
+        XPage page = getPage(zOffset);
         BitMap bitMap = getOrAddBitMap(page);
         int index = pageLocalBitMapIndex(page, zOffset);
 
@@ -95,7 +95,7 @@ public class ZExternalBitMap implements BitMapInterface {
     @Override
     public void atPut(long offset, boolean value) {
         long zOffset = convertToZOffset(offset);
-        ZPage page = getPage(zOffset);
+        XPage page = getPage(zOffset);
         BitMap bitMap = getOrAddBitMap(page);
         int index = pageLocalBitMapIndex(page, zOffset);
 
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwarding.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XForwarding.java
similarity index 76%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwarding.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XForwarding.java
index f523ff3e3a9..727014a8479 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwarding.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XForwarding.java
@@ -23,7 +23,7 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import java.util.Iterator;
 
@@ -35,7 +35,7 @@ import sun.jvm.hotspot.types.CIntegerField;
 import sun.jvm.hotspot.types.Type;
 import sun.jvm.hotspot.types.TypeDataBase;
 
-public class ZForwarding extends VMObject {
+public class XForwarding extends VMObject {
     private static Type type;
     private static long virtualFieldOffset;
     private static long entriesFieldOffset;
@@ -47,7 +47,7 @@ public class ZForwarding extends VMObject {
     }
 
     private static synchronized void initialize(TypeDataBase db) {
-        type = db.lookupType("ZForwarding");
+        type = db.lookupType("XForwarding");
 
         virtualFieldOffset = type.getField("_virtual").getOffset();
         entriesFieldOffset = type.getField("_entries").getOffset();
@@ -55,7 +55,7 @@ public class ZForwarding extends VMObject {
         refCountField = type.getCIntegerField("_ref_count");
     }
 
-    public ZForwarding(Address addr) {
+    public XForwarding(Address addr) {
         super(addr);
     }
 
@@ -63,12 +63,12 @@ public class ZForwarding extends VMObject {
         return type.getSize();
     }
 
-    private ZVirtualMemory virtual() {
-        return VMObjectFactory.newObject(ZVirtualMemory.class, addr.addOffsetTo(virtualFieldOffset));
+    private XVirtualMemory virtual() {
+        return VMObjectFactory.newObject(XVirtualMemory.class, addr.addOffsetTo(virtualFieldOffset));
     }
 
-    private ZAttachedArrayForForwarding entries() {
-        return VMObjectFactory.newObject(ZAttachedArrayForForwarding.class, addr.addOffsetTo(entriesFieldOffset));
+    private XAttachedArrayForForwarding entries() {
+        return VMObjectFactory.newObject(XAttachedArrayForForwarding.class, addr.addOffsetTo(entriesFieldOffset));
     }
 
     public long start() {
@@ -83,21 +83,21 @@ public class ZForwarding extends VMObject {
         return refCountField.getValue(addr) > 0;
     }
 
-    private ZForwardingEntry at(long cursor) {
-        long offset = ZForwardingEntry.getSize() * cursor;
+    private XForwardingEntry at(long cursor) {
+        long offset = XForwardingEntry.getSize() * cursor;
         Address entryAddress = entries().get(this).getAddress().addOffsetTo(offset);
-        return VMObjectFactory.newObject(ZForwardingEntry.class, entryAddress);
+        return VMObjectFactory.newObject(XForwardingEntry.class, entryAddress);
     }
 
-    private class ZForwardEntryIterator implements Iterator<ZForwardingEntry> {
+    private class XForwardEntryIterator implements Iterator<XForwardingEntry> {
 
         private long cursor;
 
-        private ZForwardingEntry nextEntry;
+        private XForwardingEntry nextEntry;
 
-        public ZForwardEntryIterator(long fromIndex) {
+        public XForwardEntryIterator(long fromIndex) {
             long mask = entries().length() - 1;
-            long hash = ZHash.uint32_to_uint32(fromIndex);
+            long hash = XHash.uint32_to_uint32(fromIndex);
             cursor = hash & mask;
             nextEntry = at(cursor);
         }
@@ -108,8 +108,8 @@ public class ZForwarding extends VMObject {
         }
 
         @Override
-        public ZForwardingEntry next() {
-            ZForwardingEntry entry = nextEntry;
+        public XForwardingEntry next() {
+            XForwardingEntry entry = nextEntry;
 
             long mask = entries().length() - 1;
             cursor = (cursor + 1) & mask;
@@ -118,15 +118,15 @@ public class ZForwarding extends VMObject {
             return entry;
         }
 
-        public ZForwardingEntry peak() {
+        public XForwardingEntry peak() {
             return nextEntry;
         }
     }
 
-    public ZForwardingEntry find(long fromIndex) {
-        ZForwardEntryIterator itr = new ZForwardEntryIterator(fromIndex);
+    public XForwardingEntry find(long fromIndex) {
+        XForwardEntryIterator itr = new XForwardEntryIterator(fromIndex);
         while (itr.hasNext()) {
-            ZForwardingEntry entry = itr.next();
+            XForwardingEntry entry = itr.next();
             if (entry.fromIndex() == fromIndex) {
                 return entry;
             }
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwardingEntry.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XForwardingEntry.java
similarity index 88%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwardingEntry.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XForwardingEntry.java
index 06be593424c..aa4b55775ec 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwardingEntry.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XForwardingEntry.java
@@ -23,7 +23,7 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import sun.jvm.hotspot.debugger.Address;
 import sun.jvm.hotspot.runtime.VM;
@@ -33,7 +33,7 @@ import sun.jvm.hotspot.types.CIntegerField;
 import sun.jvm.hotspot.types.Type;
 import sun.jvm.hotspot.types.TypeDataBase;
 
-public class ZForwardingEntry extends VMObject {
+public class XForwardingEntry extends VMObject {
     private static Type type;
     private static CIntegerField entryField;
 
@@ -42,7 +42,7 @@ public class ZForwardingEntry extends VMObject {
     }
 
     private static synchronized void initialize(TypeDataBase db) {
-        type = db.lookupType("ZForwardingEntry");
+        type = db.lookupType("XForwardingEntry");
 
         entryField = type.getCIntegerField("_entry");
     }
@@ -51,7 +51,7 @@ public class ZForwardingEntry extends VMObject {
         return type.getSize();
     }
 
-    public ZForwardingEntry(Address addr) {
+    public XForwardingEntry(Address addr) {
         super(addr);
     }
 
@@ -59,7 +59,7 @@ public class ZForwardingEntry extends VMObject {
         return entryField.getValue(addr);
     }
 
-    // typedef ZBitField<uint64_t, bool,   0,   1> field_populated
+    // typedef XBitField<uint64_t, bool,   0,   1> field_populated
     private boolean fieldPopulatedDecode(long value) {
         long FieldMask = (1L << 1) - 1;
         int FieldShift = 1;
@@ -67,7 +67,7 @@ public class ZForwardingEntry extends VMObject {
         return (((value >>> FieldShift) & FieldMask) << ValueShift) != 0L;
     }
 
-    // typedef ZBitField<uint64_t, size_t, 1,  45> field_to_offset;
+    // typedef XBitField<uint64_t, size_t, 1,  45> field_to_offset;
     private long fieldToOffsetDecode(long value) {
         long FieldMask = (1L << 45) - 1;
         int FieldShift = 1;
@@ -75,7 +75,7 @@ public class ZForwardingEntry extends VMObject {
         return ((value >>> FieldShift) & FieldMask) << ValueShift;
     }
 
-    // typedef ZBitField<uint64_t, size_t, 46, 18> field_from_index;
+    // typedef XBitField<uint64_t, size_t, 46, 18> field_from_index;
     private long fieldFromIndexDecode(long value) {
         long FieldMask = (1L << 18) - 1;
         int FieldShift = 46;
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwardingTable.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XForwardingTable.java
similarity index 80%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwardingTable.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XForwardingTable.java
index a1cd2447688..259f48a37b6 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwardingTable.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XForwardingTable.java
@@ -22,7 +22,7 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import sun.jvm.hotspot.debugger.Address;
 import sun.jvm.hotspot.runtime.VM;
@@ -33,7 +33,7 @@ import sun.jvm.hotspot.types.CIntegerField;
 import sun.jvm.hotspot.types.Type;
 import sun.jvm.hotspot.types.TypeDataBase;
 
-public class ZForwardingTable extends VMObject {
+public class XForwardingTable extends VMObject {
     private static long mapFieldOffset;
 
     static {
@@ -41,20 +41,20 @@ public class ZForwardingTable extends VMObject {
     }
 
     private static synchronized void initialize(TypeDataBase db) {
-        Type type = db.lookupType("ZForwardingTable");
+        Type type = db.lookupType("XForwardingTable");
 
         mapFieldOffset = type.getAddressField("_map").getOffset();
     }
 
-    public ZForwardingTable(Address addr) {
+    public XForwardingTable(Address addr) {
         super(addr);
     }
 
-    private ZGranuleMapForForwarding map() {
-        return VMObjectFactory.newObject(ZGranuleMapForForwarding.class, addr.addOffsetTo(mapFieldOffset));
+    private XGranuleMapForForwarding map() {
+        return VMObjectFactory.newObject(XGranuleMapForForwarding.class, addr.addOffsetTo(mapFieldOffset));
     }
 
-    public ZForwarding get(Address o) {
-        return VMObjectFactory.newObject(ZForwarding.class, map().get(ZAddress.offset(o)));
+    public XForwarding get(Address o) {
+        return VMObjectFactory.newObject(XForwarding.class, map().get(XAddress.offset(o)));
     }
 }
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwardingTableCursor.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XForwardingTableCursor.java
similarity index 94%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwardingTableCursor.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XForwardingTableCursor.java
index 034e3b6ab8a..40103cd1e43 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwardingTableCursor.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XForwardingTableCursor.java
@@ -22,8 +22,8 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
-class ZForwardingTableCursor {
+class XForwardingTableCursor {
     long _value;
 }
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwardingTableEntry.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XForwardingTableEntry.java
similarity index 93%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwardingTableEntry.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XForwardingTableEntry.java
index 52b57699e5a..7c629f7c5cf 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZForwardingTableEntry.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XForwardingTableEntry.java
@@ -22,14 +22,14 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import sun.jvm.hotspot.debugger.Address;
 
-class ZForwardingTableEntry {
+class XForwardingTableEntry {
     private Address entry;
 
-    ZForwardingTableEntry(Address addr) {
+    XForwardingTableEntry(Address addr) {
         entry = addr;
     }
 
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XGlobals.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XGlobals.java
new file mode 100644
index 00000000000..1cfc6dd7663
--- /dev/null
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XGlobals.java
@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+package sun.jvm.hotspot.gc.x;
+
+import sun.jvm.hotspot.runtime.VM;
+import sun.jvm.hotspot.types.Field;
+import sun.jvm.hotspot.types.Type;
+import sun.jvm.hotspot.types.TypeDataBase;
+
+public class XGlobals {
+    private static Field instanceField;
+
+    // Global phase state
+    public static int XPhaseRelocate;
+
+    public static byte XPageTypeSmall;
+    public static byte XPageTypeMedium;
+    public static byte XPageTypeLarge;
+
+    // Granule size shift
+    public static long XGranuleSizeShift;
+
+    // Page size shifts
+    public static long XPageSizeSmallShift;
+    public static long XPageSizeMediumShift;
+
+    // Object alignment shifts
+    public static int  XObjectAlignmentMediumShift;
+    public static int  XObjectAlignmentLargeShift;
+
+    // Pointer part of address
+    public static long XAddressOffsetShift;
+
+    // Pointer part of address
+    public static long XAddressOffsetBits;
+    public static long XAddressOffsetMax;
+
+    static {
+        VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase()));
+    }
+
+    private static synchronized void initialize(TypeDataBase db) {
+        Type type = db.lookupType("XGlobalsForVMStructs");
+
+        instanceField = type.getField("_instance_p");
+
+        XPhaseRelocate = db.lookupIntConstant("XPhaseRelocate").intValue();
+
+        XPageTypeSmall = db.lookupIntConstant("XPageTypeSmall").byteValue();
+        XPageTypeMedium = db.lookupIntConstant("XPageTypeMedium").byteValue();
+        XPageTypeLarge = db.lookupIntConstant("XPageTypeLarge").byteValue();
+
+        XGranuleSizeShift = db.lookupLongConstant("XGranuleSizeShift").longValue();
+
+        XPageSizeSmallShift = db.lookupLongConstant("XPageSizeSmallShift").longValue();
+        XPageSizeMediumShift = db.lookupLongConstant("XPageSizeMediumShift").longValue();
+
+        XObjectAlignmentMediumShift = db.lookupIntConstant("XObjectAlignmentMediumShift").intValue();
+        XObjectAlignmentLargeShift = db.lookupIntConstant("XObjectAlignmentLargeShift").intValue();
+
+        XAddressOffsetShift = db.lookupLongConstant("XAddressOffsetShift").longValue();
+
+        XAddressOffsetBits = db.lookupLongConstant("XAddressOffsetBits").longValue();
+        XAddressOffsetMax  = db.lookupLongConstant("XAddressOffsetMax").longValue();
+    }
+
+    private static XGlobalsForVMStructs instance() {
+        return new XGlobalsForVMStructs(instanceField.getAddress());
+    }
+
+    public static int XGlobalPhase() {
+        return instance().XGlobalPhase();
+    }
+
+    public static int XGlobalSeqNum() {
+        return instance().XGlobalSeqNum();
+    }
+
+    public static long XAddressOffsetMask() {
+        return instance().XAddressOffsetMask();
+    }
+
+    public static long XAddressMetadataMask() {
+        return instance().XAddressMetadataMask();
+    }
+
+    public static long XAddressMetadataFinalizable() {
+        return instance().XAddressMetadataFinalizable();
+    }
+
+    public static long XAddressGoodMask() {
+        return instance().XAddressGoodMask();
+    }
+
+    public static long XAddressBadMask() {
+        return instance().XAddressBadMask();
+    }
+
+    public static long XAddressWeakBadMask() {
+        return instance().XAddressWeakBadMask();
+    }
+
+    public static int XObjectAlignmentSmallShift() {
+        return instance().XObjectAlignmentSmallShift();
+    }
+
+    public static int XObjectAlignmentSmall() {
+        return instance().XObjectAlignmentSmall();
+    }
+}
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XGlobalsForVMStructs.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XGlobalsForVMStructs.java
new file mode 100644
index 00000000000..d4930dcd5dc
--- /dev/null
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XGlobalsForVMStructs.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+package sun.jvm.hotspot.gc.x;
+
+import sun.jvm.hotspot.debugger.Address;
+import sun.jvm.hotspot.runtime.VM;
+import sun.jvm.hotspot.runtime.VMObject;
+import sun.jvm.hotspot.types.AddressField;
+import sun.jvm.hotspot.types.Type;
+import sun.jvm.hotspot.types.TypeDataBase;
+
+class XGlobalsForVMStructs extends VMObject {
+    private static AddressField XGlobalPhaseField;
+    private static AddressField XGlobalSeqNumField;
+    private static AddressField XAddressOffsetMaskField;
+    private static AddressField XAddressMetadataMaskField;
+    private static AddressField XAddressMetadataFinalizableField;
+    private static AddressField XAddressGoodMaskField;
+    private static AddressField XAddressBadMaskField;
+    private static AddressField XAddressWeakBadMaskField;
+    private static AddressField XObjectAlignmentSmallShiftField;
+    private static AddressField XObjectAlignmentSmallField;
+
+    static {
+        VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase()));
+    }
+
+    private static synchronized void initialize(TypeDataBase db) {
+        Type type = db.lookupType("XGlobalsForVMStructs");
+
+        XGlobalPhaseField = type.getAddressField("_XGlobalPhase");
+        XGlobalSeqNumField = type.getAddressField("_XGlobalSeqNum");
+        XAddressOffsetMaskField = type.getAddressField("_XAddressOffsetMask");
+        XAddressMetadataMaskField = type.getAddressField("_XAddressMetadataMask");
+        XAddressMetadataFinalizableField = type.getAddressField("_XAddressMetadataFinalizable");
+        XAddressGoodMaskField = type.getAddressField("_XAddressGoodMask");
+        XAddressBadMaskField = type.getAddressField("_XAddressBadMask");
+        XAddressWeakBadMaskField = type.getAddressField("_XAddressWeakBadMask");
+        XObjectAlignmentSmallShiftField = type.getAddressField("_XObjectAlignmentSmallShift");
+        XObjectAlignmentSmallField = type.getAddressField("_XObjectAlignmentSmall");
+    }
+
+    XGlobalsForVMStructs(Address addr) {
+        super(addr);
+    }
+
+    int XGlobalPhase() {
+        return XGlobalPhaseField.getValue(addr).getJIntAt(0);
+    }
+
+    int XGlobalSeqNum() {
+        return XGlobalSeqNumField.getValue(addr).getJIntAt(0);
+    }
+
+    long XAddressOffsetMask() {
+        return XAddressOffsetMaskField.getValue(addr).getJLongAt(0);
+    }
+
+    long XAddressMetadataMask() {
+        return XAddressMetadataMaskField.getValue(addr).getJLongAt(0);
+    }
+
+    long XAddressMetadataFinalizable() {
+        return XAddressMetadataFinalizableField.getValue(addr).getJLongAt(0);
+    }
+
+    long XAddressGoodMask() {
+        return XAddressGoodMaskField.getValue(addr).getJLongAt(0);
+    }
+
+    long XAddressBadMask() {
+        return XAddressBadMaskField.getValue(addr).getJLongAt(0);
+    }
+
+    long XAddressWeakBadMask() {
+        return XAddressWeakBadMaskField.getValue(addr).getJLongAt(0);
+    }
+
+    int XObjectAlignmentSmallShift() {
+        return XObjectAlignmentSmallShiftField.getValue(addr).getJIntAt(0);
+    }
+
+    int XObjectAlignmentSmall() {
+        return XObjectAlignmentSmallField.getValue(addr).getJIntAt(0);
+    }
+}
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGranuleMapForForwarding.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XGranuleMapForForwarding.java
similarity index 87%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGranuleMapForForwarding.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XGranuleMapForForwarding.java
index d2dc56fd881..347f1405729 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGranuleMapForForwarding.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XGranuleMapForForwarding.java
@@ -23,7 +23,7 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import sun.jvm.hotspot.debugger.Address;
 import sun.jvm.hotspot.runtime.VM;
@@ -32,7 +32,7 @@ import sun.jvm.hotspot.types.AddressField;
 import sun.jvm.hotspot.types.Type;
 import sun.jvm.hotspot.types.TypeDataBase;
 
-public class ZGranuleMapForForwarding  extends VMObject {
+public class XGranuleMapForForwarding  extends VMObject {
     private static AddressField mapField;
 
     static {
@@ -40,12 +40,12 @@ public class ZGranuleMapForForwarding  extends VMObject {
     }
 
     private static synchronized void initialize(TypeDataBase db) {
-        Type type = db.lookupType("ZGranuleMapForForwarding");
+        Type type = db.lookupType("XGranuleMapForForwarding");
 
         mapField = type.getAddressField("_map");
     }
 
-    public ZGranuleMapForForwarding(Address addr) {
+    public XGranuleMapForForwarding(Address addr) {
         super(addr);
     }
 
@@ -54,11 +54,11 @@ public class ZGranuleMapForForwarding  extends VMObject {
     }
 
     public long size() {
-        return ZGlobals.ZAddressOffsetMax >> ZGlobals.ZGranuleSizeShift;
+        return XGlobals.XAddressOffsetMax >> XGlobals.XGranuleSizeShift;
     }
 
     private long index_for_offset(long offset) {
-        long index = offset >>> ZGlobals.ZGranuleSizeShift;
+        long index = offset >>> XGlobals.XGranuleSizeShift;
 
         return index;
     }
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGranuleMapForPageTable.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XGranuleMapForPageTable.java
similarity index 87%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGranuleMapForPageTable.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XGranuleMapForPageTable.java
index e24ce4189a9..468a3e2457d 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGranuleMapForPageTable.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XGranuleMapForPageTable.java
@@ -22,7 +22,7 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import sun.jvm.hotspot.debugger.Address;
 import sun.jvm.hotspot.runtime.VM;
@@ -31,7 +31,7 @@ import sun.jvm.hotspot.types.AddressField;
 import sun.jvm.hotspot.types.Type;
 import sun.jvm.hotspot.types.TypeDataBase;
 
-public class ZGranuleMapForPageTable  extends VMObject {
+public class XGranuleMapForPageTable  extends VMObject {
     private static AddressField mapField;
 
     static {
@@ -39,12 +39,12 @@ public class ZGranuleMapForPageTable  extends VMObject {
     }
 
     private static synchronized void initialize(TypeDataBase db) {
-        Type type = db.lookupType("ZGranuleMapForPageTable");
+        Type type = db.lookupType("XGranuleMapForPageTable");
 
         mapField = type.getAddressField("_map");
     }
 
-    public ZGranuleMapForPageTable(Address addr) {
+    public XGranuleMapForPageTable(Address addr) {
         super(addr);
     }
 
@@ -53,11 +53,11 @@ public class ZGranuleMapForPageTable  extends VMObject {
     }
 
     public long size() {
-        return ZGlobals.ZAddressOffsetMax >> ZGlobals.ZGranuleSizeShift;
+        return XGlobals.XAddressOffsetMax >> XGlobals.XGranuleSizeShift;
     }
 
     private long index_for_addr(Address addr) {
-        long index = ZAddress.offset(addr) >> ZGlobals.ZGranuleSizeShift;
+        long index = XAddress.offset(addr) >> XGlobals.XGranuleSizeShift;
 
         return index;
     }
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZHash.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XHash.java
similarity index 97%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZHash.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XHash.java
index e8e0bb9990b..79b1f5c2e70 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZHash.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XHash.java
@@ -22,9 +22,9 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
-class ZHash {
+class XHash {
     private static long uint32(long value) {
         return value & 0xFFFFFFFFL;
     }
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XHeap.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XHeap.java
new file mode 100644
index 00000000000..c309ce21944
--- /dev/null
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XHeap.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+package sun.jvm.hotspot.gc.x;
+
+import java.io.PrintStream;
+
+import sun.jvm.hotspot.debugger.Address;
+import sun.jvm.hotspot.runtime.VM;
+import sun.jvm.hotspot.runtime.VMObject;
+import sun.jvm.hotspot.runtime.VMObjectFactory;
+import sun.jvm.hotspot.types.Type;
+import sun.jvm.hotspot.types.TypeDataBase;
+
+// Mirror class for XHeap
+
+public class XHeap extends VMObject {
+
+    private static long pageAllocatorFieldOffset;
+    private static long pageTableFieldOffset;
+    private static long forwardingTableFieldOffset;
+    private static long relocateFieldOffset;
+
+    static {
+        VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase()));
+    }
+
+    private static synchronized void initialize(TypeDataBase db) {
+        Type type = db.lookupType("XHeap");
+
+        pageAllocatorFieldOffset = type.getAddressField("_page_allocator").getOffset();
+        pageTableFieldOffset = type.getAddressField("_page_table").getOffset();
+        forwardingTableFieldOffset = type.getAddressField("_forwarding_table").getOffset();
+        relocateFieldOffset = type.getAddressField("_relocate").getOffset();
+    }
+
+    public XHeap(Address addr) {
+        super(addr);
+    }
+
+    private XPageAllocator pageAllocator() {
+        Address pageAllocatorAddr = addr.addOffsetTo(pageAllocatorFieldOffset);
+        return VMObjectFactory.newObject(XPageAllocator.class, pageAllocatorAddr);
+    }
+
+    XPageTable pageTable() {
+        return VMObjectFactory.newObject(XPageTable.class, addr.addOffsetTo(pageTableFieldOffset));
+    }
+
+    XForwardingTable forwardingTable() {
+        return VMObjectFactory.newObject(XForwardingTable.class, addr.addOffsetTo(forwardingTableFieldOffset));
+    }
+
+    XRelocate relocate() {
+        return VMObjectFactory.newObject(XRelocate.class, addr.addOffsetTo(relocateFieldOffset));
+    }
+
+    public long maxCapacity() {
+        return pageAllocator().maxCapacity();
+    }
+
+    public long capacity() {
+        return pageAllocator().capacity();
+    }
+
+    public long used() {
+        return pageAllocator().used();
+    }
+
+    boolean is_relocating(Address o) {
+        return pageTable().is_relocating(o);
+    }
+
+    Address relocate_object(Address addr) {
+        XForwarding forwarding = forwardingTable().get(addr);
+        if (forwarding == null) {
+            return XAddress.good(addr);
+        }
+        return relocate().relocateObject(forwarding, XAddress.good(addr));
+    }
+
+    public boolean isIn(Address addr) {
+        if (XAddress.isIn(addr)) {
+            XPage page = pageTable().get(addr);
+            if (page != null) {
+                return page.isIn(addr);
+            }
+        }
+        return false;
+    }
+
+    public Address remapObject(Address o) {
+        XForwarding forwarding = forwardingTable().get(addr);
+        if (forwarding == null) {
+            return XAddress.good(o);
+        }
+        return relocate().forwardObject(forwarding, XAddress.good(o));
+    }
+
+    public void printOn(PrintStream tty) {
+        tty.print(" ZHeap          ");
+        tty.print("used " + (used() / 1024 / 1024) + "M, ");
+        tty.print("capacity " + (capacity() / 1024 / 1024) + "M, ");
+        tty.println("max capacity " + (maxCapacity() / 1024 / 1024) + "M");
+    }
+}
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZOop.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XOop.java
similarity index 96%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZOop.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XOop.java
index ae8d812003e..bbe296f658b 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZOop.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XOop.java
@@ -22,12 +22,12 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import sun.jvm.hotspot.debugger.Address;
 import sun.jvm.hotspot.debugger.OopHandle;
 
-class ZOop {
+class XOop {
     static Address to_address(OopHandle oop) {
         return oop;
     }
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPage.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XPage.java
similarity index 83%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPage.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XPage.java
index f75d9bc9869..a6315f10130 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPage.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XPage.java
@@ -22,7 +22,7 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -41,7 +41,7 @@ import sun.jvm.hotspot.types.CIntegerField;
 import sun.jvm.hotspot.types.Type;
 import sun.jvm.hotspot.types.TypeDataBase;
 
-public class ZPage extends VMObject implements LiveRegionsProvider {
+public class XPage extends VMObject implements LiveRegionsProvider {
     private static CIntegerField typeField;
     private static CIntegerField seqnumField;
     private static long virtualFieldOffset;
@@ -52,7 +52,7 @@ public class ZPage extends VMObject implements LiveRegionsProvider {
     }
 
     private static synchronized void initialize(TypeDataBase db) {
-        Type type = db.lookupType("ZPage");
+        Type type = db.lookupType("XPage");
 
         typeField = type.getCIntegerField("_type");
         seqnumField = type.getCIntegerField("_seqnum");
@@ -60,7 +60,7 @@ public class ZPage extends VMObject implements LiveRegionsProvider {
         topField = type.getAddressField("_top");
     }
 
-    public ZPage(Address addr) {
+    public XPage(Address addr) {
         super(addr);
     }
 
@@ -72,8 +72,8 @@ public class ZPage extends VMObject implements LiveRegionsProvider {
         return seqnumField.getJInt(addr);
     }
 
-    private ZVirtualMemory virtual() {
-        return VMObjectFactory.newObject(ZVirtualMemory.class, addr.addOffsetTo(virtualFieldOffset));
+    private XVirtualMemory virtual() {
+        return VMObjectFactory.newObject(XVirtualMemory.class, addr.addOffsetTo(virtualFieldOffset));
     }
 
     private Address top() {
@@ -81,7 +81,7 @@ public class ZPage extends VMObject implements LiveRegionsProvider {
     }
 
     private boolean is_relocatable() {
-        return seqnum() < ZGlobals.ZGlobalSeqNum();
+        return seqnum() < XGlobals.XGlobalSeqNum();
     }
 
     long start() {
@@ -93,13 +93,13 @@ public class ZPage extends VMObject implements LiveRegionsProvider {
     }
 
     long object_alignment_shift() {
-        if (type() == ZGlobals.ZPageTypeSmall) {
-            return ZGlobals.ZObjectAlignmentSmallShift();
-        } else if (type() == ZGlobals.ZPageTypeMedium) {
-            return ZGlobals.ZObjectAlignmentMediumShift;
+        if (type() == XGlobals.XPageTypeSmall) {
+            return XGlobals.XObjectAlignmentSmallShift();
+        } else if (type() == XGlobals.XPageTypeMedium) {
+            return XGlobals.XObjectAlignmentMediumShift;
         } else {
-            assert(type() == ZGlobals.ZPageTypeLarge);
-            return ZGlobals.ZObjectAlignmentLargeShift;
+            assert(type() == XGlobals.XPageTypeLarge);
+            return XGlobals.XObjectAlignmentLargeShift;
         }
     }
 
@@ -108,7 +108,7 @@ public class ZPage extends VMObject implements LiveRegionsProvider {
     }
 
     public boolean isIn(Address addr) {
-        long offset = ZAddress.offset(addr);
+        long offset = XAddress.offset(addr);
         // FIXME: it does not consider the sign.
         return (offset >= start()) && (offset < top().asLongValue());
     }
@@ -127,7 +127,7 @@ public class ZPage extends VMObject implements LiveRegionsProvider {
     }
 
     public List<MemRegion> getLiveRegions() {
-        Address start = ZAddress.good(ZUtils.longToAddress(start()));
+        Address start = XAddress.good(XUtils.longToAddress(start()));
 
         // Can't convert top() to a "good" address because it might
         // be at the top of the "offset" range, and therefore also
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XPageAllocator.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XPageAllocator.java
new file mode 100644
index 00000000000..1af19ea875f
--- /dev/null
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XPageAllocator.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+package sun.jvm.hotspot.gc.x;
+
+import sun.jvm.hotspot.debugger.Address;
+import sun.jvm.hotspot.runtime.VM;
+import sun.jvm.hotspot.runtime.VMObject;
+import sun.jvm.hotspot.types.CIntegerField;
+import sun.jvm.hotspot.types.Type;
+import sun.jvm.hotspot.types.TypeDataBase;
+
+// Mirror class for XPageAllocator
+
+public class XPageAllocator extends VMObject {
+
+    private static CIntegerField maxCapacityField;
+    private static CIntegerField capacityField;
+    private static CIntegerField usedField;
+
+    static {
+        VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase()));
+    }
+
+    private static synchronized void initialize(TypeDataBase db) {
+        Type type = db.lookupType("XPageAllocator");
+
+        maxCapacityField = type.getCIntegerField("_max_capacity");
+        capacityField = type.getCIntegerField("_capacity");
+        usedField = type.getCIntegerField("_used");
+    }
+
+    public long maxCapacity() {
+        return maxCapacityField.getValue(addr);
+    }
+
+    public long capacity() {
+        return capacityField.getValue(addr);
+    }
+
+    public long used() {
+        return usedField.getValue(addr);
+    }
+
+    public XPageAllocator(Address addr) {
+        super(addr);
+    }
+}
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPageTable.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XPageTable.java
similarity index 66%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPageTable.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XPageTable.java
index fdc641e38eb..c2aba43b912 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPageTable.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XPageTable.java
@@ -22,7 +22,7 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import java.util.Iterator;
 
@@ -33,7 +33,7 @@ import sun.jvm.hotspot.runtime.VMObjectFactory;
 import sun.jvm.hotspot.types.Type;
 import sun.jvm.hotspot.types.TypeDataBase;
 
-public class ZPageTable extends VMObject {
+public class XPageTable extends VMObject {
     private static long mapFieldOffset;
 
     static {
@@ -41,49 +41,49 @@ public class ZPageTable extends VMObject {
     }
 
     private static synchronized void initialize(TypeDataBase db) {
-        Type type = db.lookupType("ZPageTable");
+        Type type = db.lookupType("XPageTable");
 
         mapFieldOffset = type.getAddressField("_map").getOffset();
     }
 
-    public ZPageTable(Address addr) {
+    public XPageTable(Address addr) {
         super(addr);
     }
 
-    private ZGranuleMapForPageTable map() {
-        return VMObjectFactory.newObject(ZGranuleMapForPageTable.class, addr.addOffsetTo(mapFieldOffset));
+    private XGranuleMapForPageTable map() {
+        return VMObjectFactory.newObject(XGranuleMapForPageTable.class, addr.addOffsetTo(mapFieldOffset));
     }
 
-    private ZPageTableEntry getEntry(Address o) {
-        return new ZPageTableEntry(map().get(o));
+    private XPageTableEntry getEntry(Address o) {
+        return new XPageTableEntry(map().get(o));
     }
 
-    ZPage get(Address o) {
-        return VMObjectFactory.newObject(ZPage.class, map().get(VM.getVM().getDebugger().newAddress(ZAddress.offset(o))));
+    XPage get(Address o) {
+        return VMObjectFactory.newObject(XPage.class, map().get(VM.getVM().getDebugger().newAddress(XAddress.offset(o))));
     }
 
     boolean is_relocating(Address o) {
         return getEntry(o).relocating();
     }
 
-    private class ZPagesIterator implements Iterator<ZPage> {
-        private ZGranuleMapForPageTable.Iterator mapIter;
-        private ZPage next;
+    private class XPagesIterator implements Iterator<XPage> {
+        private XGranuleMapForPageTable.Iterator mapIter;
+        private XPage next;
 
-        ZPagesIterator() {
+        XPagesIterator() {
             mapIter = map().new Iterator();
             positionToNext();
         }
 
-        private ZPage positionToNext() {
-            ZPage current = next;
+        private XPage positionToNext() {
+            XPage current = next;
 
             // Find next
-            ZPage found = null;
+            XPage found = null;
             while (mapIter.hasNext()) {
-                ZPageTableEntry entry = new ZPageTableEntry(mapIter.next());
+                XPageTableEntry entry = new XPageTableEntry(mapIter.next());
                 if (!entry.isEmpty()) {
-                    ZPage page = entry.page();
+                    XPage page = entry.page();
                     // Medium pages have repeated entries for all covered slots,
                     // therefore we need to compare against the current page.
                     if (page != null && !page.equals(current)) {
@@ -104,7 +104,7 @@ public class ZPageTable extends VMObject {
         }
 
         @Override
-        public ZPage next() {
+        public XPage next() {
             return positionToNext();
         }
 
@@ -114,27 +114,27 @@ public class ZPageTable extends VMObject {
         }
     }
 
-    abstract class ZPageFilter {
-        public abstract boolean accept(ZPage page);
+    abstract class XPageFilter {
+        public abstract boolean accept(XPage page);
     }
 
-    class ZPagesFilteredIterator implements Iterator<ZPage> {
-        private ZPage next;
-        private ZPagesIterator iter = new ZPagesIterator();
-        private ZPageFilter filter;
+    class XPagesFilteredIterator implements Iterator<XPage> {
+        private XPage next;
+        private XPagesIterator iter = new XPagesIterator();
+        private XPageFilter filter;
 
-        ZPagesFilteredIterator(ZPageFilter filter) {
+        XPagesFilteredIterator(XPageFilter filter) {
             this.filter = filter;
             positionToNext();
         }
 
-        public ZPage positionToNext() {
-            ZPage current = next;
+        public XPage positionToNext() {
+            XPage current = next;
 
             // Find next
-            ZPage found = null;
+            XPage found = null;
             while (iter.hasNext()) {
-                ZPage page = iter.next();
+                XPage page = iter.next();
                 if (filter.accept(page)) {
                     found = page;
                     break;
@@ -152,7 +152,7 @@ public class ZPageTable extends VMObject {
         }
 
         @Override
-        public ZPage next() {
+        public XPage next() {
             return positionToNext();
         }
 
@@ -162,13 +162,13 @@ public class ZPageTable extends VMObject {
         }
     }
 
-    public Iterator<ZPage> iterator() {
-        return new ZPagesIterator();
+    public Iterator<XPage> iterator() {
+        return new XPagesIterator();
     }
 
-    public Iterator<ZPage> activePagesIterator() {
-        return new ZPagesFilteredIterator(new ZPageFilter() {
-            public boolean accept(ZPage page) {
+    public Iterator<XPage> activePagesIterator() {
+        return new XPagesFilteredIterator(new XPageFilter() {
+            public boolean accept(XPage page) {
                 return page != null;
             }
         });
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPageTableEntry.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XPageTableEntry.java
similarity index 89%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPageTableEntry.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XPageTableEntry.java
index 25aca67f21d..42a29878ecc 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZPageTableEntry.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XPageTableEntry.java
@@ -22,20 +22,20 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import sun.jvm.hotspot.debugger.Address;
 import sun.jvm.hotspot.runtime.VMObjectFactory;
 
-class ZPageTableEntry {
+class XPageTableEntry {
     Address entry;
 
-    ZPageTableEntry(Address address) {
+    XPageTableEntry(Address address) {
         entry = address;
     }
 
-    ZPage page() {
-        return VMObjectFactory.newObject(ZPage.class, zPageBits());
+    XPage page() {
+        return VMObjectFactory.newObject(XPage.class, zPageBits());
     }
 
     private Address zPageBits() {
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZRelocate.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XRelocate.java
similarity index 78%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZRelocate.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XRelocate.java
index b301a8f8c34..a4b0dc1c992 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZRelocate.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XRelocate.java
@@ -23,7 +23,7 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import sun.jvm.hotspot.debugger.Address;
 import sun.jvm.hotspot.runtime.VM;
@@ -32,36 +32,36 @@ import sun.jvm.hotspot.types.AddressField;
 import sun.jvm.hotspot.types.Type;
 import sun.jvm.hotspot.types.TypeDataBase;
 
-public class ZRelocate  extends VMObject {
+public class XRelocate  extends VMObject {
 
     static {
         VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase()));
     }
 
     private static synchronized void initialize(TypeDataBase db) {
-        Type type = db.lookupType("ZRelocate");
+        Type type = db.lookupType("XRelocate");
     }
 
-    public ZRelocate(Address addr) {
+    public XRelocate(Address addr) {
         super(addr);
     }
 
-    private long forwardingIndex(ZForwarding forwarding, Address from) {
-        long fromOffset = ZAddress.offset(from);
+    private long forwardingIndex(XForwarding forwarding, Address from) {
+        long fromOffset = XAddress.offset(from);
         return (fromOffset - forwarding.start()) >>> forwarding.objectAlignmentShift();
     }
 
-    private Address forwardingFind(ZForwarding forwarding, Address from) {
+    private Address forwardingFind(XForwarding forwarding, Address from) {
         long fromIndex = forwardingIndex(forwarding, from);
-        ZForwardingEntry entry = forwarding.find(fromIndex);
-        return entry.populated() ? ZAddress.good(VM.getVM().getDebugger().newAddress(entry.toOffset())) : null;
+        XForwardingEntry entry = forwarding.find(fromIndex);
+        return entry.populated() ? XAddress.good(VM.getVM().getDebugger().newAddress(entry.toOffset())) : null;
     }
 
-    public Address forwardObject(ZForwarding forwarding, Address from) {
+    public Address forwardObject(XForwarding forwarding, Address from) {
         return forwardingFind(forwarding, from);
     }
 
-    public Address relocateObject(ZForwarding forwarding, Address o) {
+    public Address relocateObject(XForwarding forwarding, Address o) {
         Address toAddr = forwardingFind(forwarding, o);
         if (toAddr != null) {
             // Already relocated.
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZUtils.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XUtils.java
similarity index 96%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZUtils.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XUtils.java
index 2029a71da03..a5c9ffde371 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZUtils.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XUtils.java
@@ -22,12 +22,12 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import sun.jvm.hotspot.debugger.Address;
 import sun.jvm.hotspot.runtime.VM;
 
-class ZUtils {
+class XUtils {
     static Address longToAddress(long value) {
         return VM.getVM().getDebugger().newAddress(value);
     }
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZVirtualMemory.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XVirtualMemory.java
similarity index 91%
rename from src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZVirtualMemory.java
rename to src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XVirtualMemory.java
index cd36b2b8012..de225ccdcbd 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZVirtualMemory.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/x/XVirtualMemory.java
@@ -22,7 +22,7 @@
  *
  */
 
-package sun.jvm.hotspot.gc.z;
+package sun.jvm.hotspot.gc.x;
 
 import sun.jvm.hotspot.debugger.Address;
 import sun.jvm.hotspot.runtime.VM;
@@ -31,7 +31,7 @@ import sun.jvm.hotspot.types.CIntegerField;
 import sun.jvm.hotspot.types.Type;
 import sun.jvm.hotspot.types.TypeDataBase;
 
-public class ZVirtualMemory extends VMObject {
+public class XVirtualMemory extends VMObject {
     private static CIntegerField startField;
     private static CIntegerField endField;
 
@@ -40,13 +40,13 @@ public class ZVirtualMemory extends VMObject {
     }
 
     private static synchronized void initialize(TypeDataBase db) {
-        Type type = db.lookupType("ZVirtualMemory");
+        Type type = db.lookupType("XVirtualMemory");
 
         startField = type.getCIntegerField("_start");
         endField = type.getCIntegerField("_end");
     }
 
-    public ZVirtualMemory(Address addr) {
+    public XVirtualMemory(Address addr) {
         super(addr);
     }
 
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZCollectedHeap.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZCollectedHeap.java
index 5a5e9493622..75842e963fb 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZCollectedHeap.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZCollectedHeap.java
@@ -84,16 +84,11 @@ public class ZCollectedHeap extends CollectedHeap {
 
     @Override
     public boolean isInReserved(Address a) {
-        return heap().isIn(a);
+        throw new RuntimeException("ZCollectedHeap.isInReserved not implemented");
     }
 
     private OopHandle oop_load_barrier(Address oopAddress) {
-        oopAddress = ZBarrier.weak_barrier(oopAddress);
-        if (oopAddress == null) {
-            return null;
-        }
-
-        return oopAddress.addOffsetToAsOopHandle(0);
+        throw new RuntimeException("ZCollectedHeap.oop_load_barrier not implemented");
     }
 
     @Override
@@ -113,27 +108,16 @@ public class ZCollectedHeap extends CollectedHeap {
     }
 
     public String oopAddressDescription(OopHandle handle) {
-        Address origOop = ZOop.to_address(handle);
-        Address loadBarrieredOop = ZBarrier.weak_barrier(origOop);
-        if (!origOop.equals(loadBarrieredOop)) {
-            return origOop + " (" + loadBarrieredOop.toString() + ")";
-        } else {
-            return handle.toString();
-        }
+        return handle.toString();
     }
 
     @Override
     public void liveRegionsIterate(LiveRegionsClosure closure) {
-        Iterator<ZPage> iter = heap().pageTable().activePagesIterator();
-        while (iter.hasNext()) {
-            ZPage page = iter.next();
-            closure.doLiveRegions(page);
-        }
+        throw new RuntimeException("ZCollectedHeap.liveRegionsIterate not implemented");
     }
 
     @Override
     public BitMapInterface createBitMap(long size) {
-        // Ignores the size
-        return new ZExternalBitMap(this);
+        throw new RuntimeException("ZCollectedHeap.createBitMap not implemented");
     }
 }
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobals.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobals.java
deleted file mode 100644
index 8e1c005d6a1..00000000000
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobals.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-package sun.jvm.hotspot.gc.z;
-
-import sun.jvm.hotspot.runtime.VM;
-import sun.jvm.hotspot.types.Field;
-import sun.jvm.hotspot.types.Type;
-import sun.jvm.hotspot.types.TypeDataBase;
-
-public class ZGlobals {
-    private static Field instanceField;
-
-    // Global phase state
-    public static int ZPhaseRelocate;
-
-    public static byte ZPageTypeSmall;
-    public static byte ZPageTypeMedium;
-    public static byte ZPageTypeLarge;
-
-    // Granule size shift
-    public static long ZGranuleSizeShift;
-
-    // Page size shifts
-    public static long ZPageSizeSmallShift;
-    public static long ZPageSizeMediumShift;
-
-    // Object alignment shifts
-    public static int  ZObjectAlignmentMediumShift;
-    public static int  ZObjectAlignmentLargeShift;
-
-    // Pointer part of address
-    public static long ZAddressOffsetShift;
-
-    // Pointer part of address
-    public static long ZAddressOffsetBits;
-    public static long ZAddressOffsetMax;
-
-    static {
-        VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase()));
-    }
-
-    private static synchronized void initialize(TypeDataBase db) {
-        Type type = db.lookupType("ZGlobalsForVMStructs");
-
-        instanceField = type.getField("_instance_p");
-
-        ZPhaseRelocate = db.lookupIntConstant("ZPhaseRelocate").intValue();
-
-        ZPageTypeSmall = db.lookupIntConstant("ZPageTypeSmall").byteValue();
-        ZPageTypeMedium = db.lookupIntConstant("ZPageTypeMedium").byteValue();
-        ZPageTypeLarge = db.lookupIntConstant("ZPageTypeLarge").byteValue();
-
-        ZGranuleSizeShift = db.lookupLongConstant("ZGranuleSizeShift").longValue();
-
-        ZPageSizeSmallShift = db.lookupLongConstant("ZPageSizeSmallShift").longValue();
-        ZPageSizeMediumShift = db.lookupLongConstant("ZPageSizeMediumShift").longValue();
-
-        ZObjectAlignmentMediumShift = db.lookupIntConstant("ZObjectAlignmentMediumShift").intValue();
-        ZObjectAlignmentLargeShift = db.lookupIntConstant("ZObjectAlignmentLargeShift").intValue();
-
-        ZAddressOffsetShift = db.lookupLongConstant("ZAddressOffsetShift").longValue();
-
-        ZAddressOffsetBits = db.lookupLongConstant("ZAddressOffsetBits").longValue();
-        ZAddressOffsetMax  = db.lookupLongConstant("ZAddressOffsetMax").longValue();
-    }
-
-    private static ZGlobalsForVMStructs instance() {
-        return new ZGlobalsForVMStructs(instanceField.getAddress());
-    }
-
-    public static int ZGlobalPhase() {
-        return instance().ZGlobalPhase();
-    }
-
-    public static int ZGlobalSeqNum() {
-        return instance().ZGlobalSeqNum();
-    }
-
-    public static long ZAddressOffsetMask() {
-        return instance().ZAddressOffsetMask();
-    }
-
-    public static long ZAddressMetadataMask() {
-        return instance().ZAddressMetadataMask();
-    }
-
-    public static long ZAddressMetadataFinalizable() {
-        return instance().ZAddressMetadataFinalizable();
-    }
-
-    public static long ZAddressGoodMask() {
-        return instance().ZAddressGoodMask();
-    }
-
-    public static long ZAddressBadMask() {
-        return instance().ZAddressBadMask();
-    }
-
-    public static long ZAddressWeakBadMask() {
-        return instance().ZAddressWeakBadMask();
-    }
-
-    public static int ZObjectAlignmentSmallShift() {
-        return instance().ZObjectAlignmentSmallShift();
-    }
-
-    public static int ZObjectAlignmentSmall() {
-        return instance().ZObjectAlignmentSmall();
-    }
-}
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobalsForVMStructs.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobalsForVMStructs.java
deleted file mode 100644
index 14b359d50a3..00000000000
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZGlobalsForVMStructs.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- *
- */
-
-package sun.jvm.hotspot.gc.z;
-
-import sun.jvm.hotspot.debugger.Address;
-import sun.jvm.hotspot.runtime.VM;
-import sun.jvm.hotspot.runtime.VMObject;
-import sun.jvm.hotspot.types.AddressField;
-import sun.jvm.hotspot.types.Type;
-import sun.jvm.hotspot.types.TypeDataBase;
-
-class ZGlobalsForVMStructs extends VMObject {
-    private static AddressField ZGlobalPhaseField;
-    private static AddressField ZGlobalSeqNumField;
-    private static AddressField ZAddressOffsetMaskField;
-    private static AddressField ZAddressMetadataMaskField;
-    private static AddressField ZAddressMetadataFinalizableField;
-    private static AddressField ZAddressGoodMaskField;
-    private static AddressField ZAddressBadMaskField;
-    private static AddressField ZAddressWeakBadMaskField;
-    private static AddressField ZObjectAlignmentSmallShiftField;
-    private static AddressField ZObjectAlignmentSmallField;
-
-    static {
-        VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase()));
-    }
-
-    private static synchronized void initialize(TypeDataBase db) {
-        Type type = db.lookupType("ZGlobalsForVMStructs");
-
-        ZGlobalPhaseField = type.getAddressField("_ZGlobalPhase");
-        ZGlobalSeqNumField = type.getAddressField("_ZGlobalSeqNum");
-        ZAddressOffsetMaskField = type.getAddressField("_ZAddressOffsetMask");
-        ZAddressMetadataMaskField = type.getAddressField("_ZAddressMetadataMask");
-        ZAddressMetadataFinalizableField = type.getAddressField("_ZAddressMetadataFinalizable");
-        ZAddressGoodMaskField = type.getAddressField("_ZAddressGoodMask");
-        ZAddressBadMaskField = type.getAddressField("_ZAddressBadMask");
-        ZAddressWeakBadMaskField = type.getAddressField("_ZAddressWeakBadMask");
-        ZObjectAlignmentSmallShiftField = type.getAddressField("_ZObjectAlignmentSmallShift");
-        ZObjectAlignmentSmallField = type.getAddressField("_ZObjectAlignmentSmall");
-    }
-
-    ZGlobalsForVMStructs(Address addr) {
-        super(addr);
-    }
-
-    int ZGlobalPhase() {
-        return ZGlobalPhaseField.getValue(addr).getJIntAt(0);
-    }
-
-    int ZGlobalSeqNum() {
-        return ZGlobalSeqNumField.getValue(addr).getJIntAt(0);
-    }
-
-    long ZAddressOffsetMask() {
-        return ZAddressOffsetMaskField.getValue(addr).getJLongAt(0);
-    }
-
-    long ZAddressMetadataMask() {
-        return ZAddressMetadataMaskField.getValue(addr).getJLongAt(0);
-    }
-
-    long ZAddressMetadataFinalizable() {
-        return ZAddressMetadataFinalizableField.getValue(addr).getJLongAt(0);
-    }
-
-    long ZAddressGoodMask() {
-        return ZAddressGoodMaskField.getValue(addr).getJLongAt(0);
-    }
-
-    long ZAddressBadMask() {
-        return ZAddressBadMaskField.getValue(addr).getJLongAt(0);
-    }
-
-    long ZAddressWeakBadMask() {
-        return ZAddressWeakBadMaskField.getValue(addr).getJLongAt(0);
-    }
-
-    int ZObjectAlignmentSmallShift() {
-        return ZObjectAlignmentSmallShiftField.getValue(addr).getJIntAt(0);
-    }
-
-    int ZObjectAlignmentSmall() {
-        return ZObjectAlignmentSmallField.getValue(addr).getJIntAt(0);
-    }
-}
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZHeap.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZHeap.java
index 172af97b604..c9a54aa2764 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZHeap.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/z/ZHeap.java
@@ -38,9 +38,6 @@ import sun.jvm.hotspot.types.TypeDataBase;
 public class ZHeap extends VMObject {
 
     private static long pageAllocatorFieldOffset;
-    private static long pageTableFieldOffset;
-    private static long forwardingTableFieldOffset;
-    private static long relocateFieldOffset;
 
     static {
         VM.registerVMInitializedObserver((o, d) -> initialize(VM.getVM().getTypeDataBase()));
@@ -50,32 +47,16 @@ public class ZHeap extends VMObject {
         Type type = db.lookupType("ZHeap");
 
         pageAllocatorFieldOffset = type.getAddressField("_page_allocator").getOffset();
-        pageTableFieldOffset = type.getAddressField("_page_table").getOffset();
-        forwardingTableFieldOffset = type.getAddressField("_forwarding_table").getOffset();
-        relocateFieldOffset = type.getAddressField("_relocate").getOffset();
     }
 
     public ZHeap(Address addr) {
         super(addr);
     }
-
     private ZPageAllocator pageAllocator() {
         Address pageAllocatorAddr = addr.addOffsetTo(pageAllocatorFieldOffset);
         return VMObjectFactory.newObject(ZPageAllocator.class, pageAllocatorAddr);
     }
 
-    ZPageTable pageTable() {
-        return VMObjectFactory.newObject(ZPageTable.class, addr.addOffsetTo(pageTableFieldOffset));
-    }
-
-    ZForwardingTable forwardingTable() {
-        return VMObjectFactory.newObject(ZForwardingTable.class, addr.addOffsetTo(forwardingTableFieldOffset));
-    }
-
-    ZRelocate relocate() {
-        return VMObjectFactory.newObject(ZRelocate.class, addr.addOffsetTo(relocateFieldOffset));
-    }
-
     public long maxCapacity() {
         return pageAllocator().maxCapacity();
     }
@@ -88,36 +69,6 @@ public class ZHeap extends VMObject {
         return pageAllocator().used();
     }
 
-    boolean is_relocating(Address o) {
-        return pageTable().is_relocating(o);
-    }
-
-    Address relocate_object(Address addr) {
-        ZForwarding forwarding = forwardingTable().get(addr);
-        if (forwarding == null) {
-            return ZAddress.good(addr);
-        }
-        return relocate().relocateObject(forwarding, ZAddress.good(addr));
-    }
-
-    public boolean isIn(Address addr) {
-        if (ZAddress.isIn(addr)) {
-            ZPage page = pageTable().get(addr);
-            if (page != null) {
-                return page.isIn(addr);
-            }
-        }
-        return false;
-    }
-
-    public Address remapObject(Address o) {
-        ZForwarding forwarding = forwardingTable().get(addr);
-        if (forwarding == null) {
-            return ZAddress.good(o);
-        }
-        return relocate().forwardObject(forwarding, ZAddress.good(o));
-    }
-
     public void printOn(PrintStream tty) {
         tty.print(" ZHeap          ");
         tty.print("used " + (used() / 1024 / 1024) + "M, ");
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/memory/Universe.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/memory/Universe.java
index 428ae4f789a..f2f5422ad04 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/memory/Universe.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/memory/Universe.java
@@ -36,6 +36,7 @@ import sun.jvm.hotspot.gc.parallel.ParallelScavengeHeap;
 import sun.jvm.hotspot.gc.serial.SerialHeap;
 import sun.jvm.hotspot.gc.shared.CollectedHeap;
 import sun.jvm.hotspot.gc.shenandoah.ShenandoahHeap;
+import sun.jvm.hotspot.gc.x.XCollectedHeap;
 import sun.jvm.hotspot.gc.z.ZCollectedHeap;
 import sun.jvm.hotspot.oops.Oop;
 import sun.jvm.hotspot.runtime.BasicType;
@@ -86,6 +87,7 @@ public class Universe {
     addHeapTypeIfInDB(db, ParallelScavengeHeap.class);
     addHeapTypeIfInDB(db, G1CollectedHeap.class);
     addHeapTypeIfInDB(db, EpsilonHeap.class);
+    addHeapTypeIfInDB(db, XCollectedHeap.class);
     addHeapTypeIfInDB(db, ZCollectedHeap.class);
     addHeapTypeIfInDB(db, ShenandoahHeap.class);
 
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/ObjectHeap.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/ObjectHeap.java
index 1a66e02b474..2c1ad426b6a 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/ObjectHeap.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/oops/ObjectHeap.java
@@ -37,7 +37,7 @@ import sun.jvm.hotspot.gc.epsilon.*;
 import sun.jvm.hotspot.gc.g1.*;
 import sun.jvm.hotspot.gc.shenandoah.*;
 import sun.jvm.hotspot.gc.parallel.*;
-import sun.jvm.hotspot.gc.z.*;
+import sun.jvm.hotspot.gc.x.*;
 import sun.jvm.hotspot.memory.*;
 import sun.jvm.hotspot.runtime.*;
 import sun.jvm.hotspot.types.*;
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/tools/HeapSummary.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/tools/HeapSummary.java
index 3980a7da8f7..d29e3a68811 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/tools/HeapSummary.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/tools/HeapSummary.java
@@ -32,6 +32,7 @@ import sun.jvm.hotspot.gc.parallel.*;
 import sun.jvm.hotspot.gc.serial.*;
 import sun.jvm.hotspot.gc.shenandoah.*;
 import sun.jvm.hotspot.gc.shared.*;
+import sun.jvm.hotspot.gc.x.*;
 import sun.jvm.hotspot.gc.z.*;
 import sun.jvm.hotspot.debugger.JVMDebugger;
 import sun.jvm.hotspot.memory.*;
@@ -143,6 +144,9 @@ public class HeapSummary extends Tool {
       } else if (heap instanceof EpsilonHeap) {
          EpsilonHeap eh = (EpsilonHeap) heap;
          printSpace(eh.space());
+      } else if (heap instanceof XCollectedHeap) {
+         XCollectedHeap zheap = (XCollectedHeap) heap;
+         zheap.printOn(System.out);
       } else if (heap instanceof ZCollectedHeap) {
          ZCollectedHeap zheap = (ZCollectedHeap) heap;
          zheap.printOn(System.out);
diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/utilities/HeapHprofBinWriter.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/utilities/HeapHprofBinWriter.java
index 0e929606e6e..c9ffca89ec7 100644
--- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/utilities/HeapHprofBinWriter.java
+++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/utilities/HeapHprofBinWriter.java
@@ -35,7 +35,6 @@ import sun.jvm.hotspot.memory.*;
 import sun.jvm.hotspot.oops.*;
 import sun.jvm.hotspot.runtime.*;
 import sun.jvm.hotspot.classfile.*;
-import sun.jvm.hotspot.gc.z.ZCollectedHeap;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 
diff --git a/src/jdk.jfr/share/conf/jfr/default.jfc b/src/jdk.jfr/share/conf/jfr/default.jfc
index 39593b2e3c9..e302da88433 100644
--- a/src/jdk.jfr/share/conf/jfr/default.jfc
+++ b/src/jdk.jfr/share/conf/jfr/default.jfc
@@ -422,6 +422,11 @@
       <setting name="threshold">0 ms</setting>
     </event>
 
+    <event name="jdk.GCPhaseConcurrentLevel2">
+      <setting name="enabled" control="gc-enabled-high">true</setting>
+      <setting name="threshold">0 ms</setting>
+    </event>
+
     <event name="jdk.GCReferenceStatistics">
       <setting name="enabled" control="gc-enabled-normal">true</setting>
     </event>
@@ -831,6 +836,16 @@
       <setting name="threshold">0 ms</setting>
     </event>
 
+    <event name="jdk.ZYoungGarbageCollection">
+      <setting name="enabled">true</setting>
+      <setting name="threshold">0 ms</setting>
+    </event>
+
+    <event name="jdk.ZOldGarbageCollection">
+      <setting name="enabled">true</setting>
+      <setting name="threshold">0 ms</setting>
+    </event>
+
     <event name="jdk.Deoptimization">
       <setting name="enabled">true</setting>
       <setting name="stackTrace">false</setting>
diff --git a/src/jdk.jfr/share/conf/jfr/profile.jfc b/src/jdk.jfr/share/conf/jfr/profile.jfc
index d15126986ef..841f20e4782 100644
--- a/src/jdk.jfr/share/conf/jfr/profile.jfc
+++ b/src/jdk.jfr/share/conf/jfr/profile.jfc
@@ -422,6 +422,11 @@
       <setting name="threshold">0 ms</setting>
     </event>
 
+    <event name="jdk.GCPhaseConcurrentLevel2">
+      <setting name="enabled" control="gc-enabled-high">true</setting>
+      <setting name="threshold">0 ms</setting>
+    </event>
+
     <event name="jdk.GCReferenceStatistics">
       <setting name="enabled" control="gc-enabled-normal">true</setting>
     </event>
@@ -831,6 +836,16 @@
       <setting name="threshold">0 ms</setting>
     </event>
 
+    <event name="jdk.ZYoungGarbageCollection">
+      <setting name="enabled">true</setting>
+      <setting name="threshold">0 ms</setting>
+    </event>
+
+    <event name="jdk.ZOldGarbageCollection">
+      <setting name="enabled">true</setting>
+      <setting name="threshold">0 ms</setting>
+    </event>
+
     <event name="jdk.Deoptimization">
       <setting name="enabled">true</setting>
       <setting name="stackTrace">true</setting>
diff --git a/test/hotspot/gtest/gc/x/test_xAddress.cpp b/test/hotspot/gtest/gc/x/test_xAddress.cpp
new file mode 100644
index 00000000000..4307cc53b3b
--- /dev/null
+++ b/test/hotspot/gtest/gc/x/test_xAddress.cpp
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "unittest.hpp"
+
+class XAddressTest : public ::testing::Test {
+protected:
+  static void is_good_bit(uintptr_t bit_mask) {
+    // Setup
+    XAddress::initialize();
+    XAddress::set_good_mask(bit_mask);
+
+    // Test that a pointer with only the given bit is considered good.
+    EXPECT_EQ(XAddress::is_good(XAddressMetadataMarked0),  (bit_mask == XAddressMetadataMarked0));
+    EXPECT_EQ(XAddress::is_good(XAddressMetadataMarked1),  (bit_mask == XAddressMetadataMarked1));
+    EXPECT_EQ(XAddress::is_good(XAddressMetadataRemapped), (bit_mask == XAddressMetadataRemapped));
+
+    // Test that a pointer with the given bit and some extra bits is considered good.
+    EXPECT_EQ(XAddress::is_good(XAddressMetadataMarked0  | 0x8),(bit_mask == XAddressMetadataMarked0));
+    EXPECT_EQ(XAddress::is_good(XAddressMetadataMarked1  | 0x8), (bit_mask == XAddressMetadataMarked1));
+    EXPECT_EQ(XAddress::is_good(XAddressMetadataRemapped | 0x8), (bit_mask == XAddressMetadataRemapped));
+
+    // Test that null is not considered good.
+    EXPECT_FALSE(XAddress::is_good(0));
+  }
+
+  static void is_good_or_null_bit(uintptr_t bit_mask) {
+    // Setup
+    XAddress::initialize();
+    XAddress::set_good_mask(bit_mask);
+
+    // Test that a pointer with only the given bit is considered good.
+    EXPECT_EQ(XAddress::is_good_or_null(XAddressMetadataMarked0),  (bit_mask == XAddressMetadataMarked0));
+    EXPECT_EQ(XAddress::is_good_or_null(XAddressMetadataMarked1),  (bit_mask == XAddressMetadataMarked1));
+    EXPECT_EQ(XAddress::is_good_or_null(XAddressMetadataRemapped), (bit_mask == XAddressMetadataRemapped));
+
+    // Test that a pointer with the given bit and some extra bits is considered good.
+    EXPECT_EQ(XAddress::is_good_or_null(XAddressMetadataMarked0  | 0x8), (bit_mask == XAddressMetadataMarked0));
+    EXPECT_EQ(XAddress::is_good_or_null(XAddressMetadataMarked1  | 0x8), (bit_mask == XAddressMetadataMarked1));
+    EXPECT_EQ(XAddress::is_good_or_null(XAddressMetadataRemapped | 0x8), (bit_mask == XAddressMetadataRemapped));
+
+    // Test that null is considered good_or_null.
+    EXPECT_TRUE(XAddress::is_good_or_null(0));
+  }
+
+  static void finalizable() {
+    // Setup
+    XAddress::initialize();
+    XAddress::flip_to_marked();
+
+    // Test that a normal good pointer is good and weak good, but not finalizable
+    const uintptr_t addr1 = XAddress::good(1);
+    EXPECT_FALSE(XAddress::is_finalizable(addr1));
+    EXPECT_TRUE(XAddress::is_marked(addr1));
+    EXPECT_FALSE(XAddress::is_remapped(addr1));
+    EXPECT_TRUE(XAddress::is_weak_good(addr1));
+    EXPECT_TRUE(XAddress::is_weak_good_or_null(addr1));
+    EXPECT_TRUE(XAddress::is_good(addr1));
+    EXPECT_TRUE(XAddress::is_good_or_null(addr1));
+
+    // Test that a finalizable good pointer is finalizable and weak good, but not good
+    const uintptr_t addr2 = XAddress::finalizable_good(1);
+    EXPECT_TRUE(XAddress::is_finalizable(addr2));
+    EXPECT_TRUE(XAddress::is_marked(addr2));
+    EXPECT_FALSE(XAddress::is_remapped(addr2));
+    EXPECT_TRUE(XAddress::is_weak_good(addr2));
+    EXPECT_TRUE(XAddress::is_weak_good_or_null(addr2));
+    EXPECT_FALSE(XAddress::is_good(addr2));
+    EXPECT_FALSE(XAddress::is_good_or_null(addr2));
+
+    // Flip to remapped and test that it's no longer weak good
+    XAddress::flip_to_remapped();
+    EXPECT_TRUE(XAddress::is_finalizable(addr2));
+    EXPECT_TRUE(XAddress::is_marked(addr2));
+    EXPECT_FALSE(XAddress::is_remapped(addr2));
+    EXPECT_FALSE(XAddress::is_weak_good(addr2));
+    EXPECT_FALSE(XAddress::is_weak_good_or_null(addr2));
+    EXPECT_FALSE(XAddress::is_good(addr2));
+    EXPECT_FALSE(XAddress::is_good_or_null(addr2));
+  }
+};
+
+TEST_F(XAddressTest, is_good) {
+  is_good_bit(XAddressMetadataMarked0);
+  is_good_bit(XAddressMetadataMarked1);
+  is_good_bit(XAddressMetadataRemapped);
+}
+
+TEST_F(XAddressTest, is_good_or_null) {
+  is_good_or_null_bit(XAddressMetadataMarked0);
+  is_good_or_null_bit(XAddressMetadataMarked1);
+  is_good_or_null_bit(XAddressMetadataRemapped);
+}
+
+TEST_F(XAddressTest, is_weak_good_or_null) {
+#define check_is_weak_good_or_null(value)                                        \
+  EXPECT_EQ(XAddress::is_weak_good_or_null(value),                               \
+            (XAddress::is_good_or_null(value) || XAddress::is_remapped(value)))  \
+    << "is_good_or_null: " << XAddress::is_good_or_null(value)                   \
+    << " is_remaped: " << XAddress::is_remapped(value)                           \
+    << " is_good_or_null_or_remapped: " << XAddress::is_weak_good_or_null(value)
+
+  check_is_weak_good_or_null((uintptr_t)NULL);
+  check_is_weak_good_or_null(XAddressMetadataMarked0);
+  check_is_weak_good_or_null(XAddressMetadataMarked1);
+  check_is_weak_good_or_null(XAddressMetadataRemapped);
+  check_is_weak_good_or_null((uintptr_t)0x123);
+}
+
+TEST_F(XAddressTest, finalizable) {
+  finalizable();
+}
diff --git a/test/hotspot/gtest/gc/x/test_xArray.cpp b/test/hotspot/gtest/gc/x/test_xArray.cpp
new file mode 100644
index 00000000000..36c0b73ad6f
--- /dev/null
+++ b/test/hotspot/gtest/gc/x/test_xArray.cpp
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2017, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xArray.inline.hpp"
+#include "unittest.hpp"
+
+TEST(XArray, sanity) {
+  XArray<int> a;
+
+  // Add elements
+  for (int i = 0; i < 10; i++) {
+    a.append(i);
+  }
+
+  XArray<int> b;
+
+  b.swap(&a);
+
+  // Check size
+  ASSERT_EQ(a.length(), 0);
+  ASSERT_EQ(a.capacity(), 0);
+  ASSERT_EQ(a.is_empty(), true);
+
+  ASSERT_EQ(b.length(), 10);
+  ASSERT_GE(b.capacity(), 10);
+  ASSERT_EQ(b.is_empty(), false);
+
+  // Clear elements
+  a.clear();
+
+  // Check that b is unaffected
+  ASSERT_EQ(b.length(), 10);
+  ASSERT_GE(b.capacity(), 10);
+  ASSERT_EQ(b.is_empty(), false);
+
+  a.append(1);
+
+  // Check that b is unaffected
+  ASSERT_EQ(b.length(), 10);
+  ASSERT_GE(b.capacity(), 10);
+  ASSERT_EQ(b.is_empty(), false);
+}
+
+TEST(XArray, iterator) {
+  XArray<int> a;
+
+  // Add elements
+  for (int i = 0; i < 10; i++) {
+    a.append(i);
+  }
+
+  // Iterate
+  int count = 0;
+  XArrayIterator<int> iter(&a);
+  for (int value; iter.next(&value);) {
+    ASSERT_EQ(a.at(count), count);
+    count++;
+  }
+
+  // Check count
+  ASSERT_EQ(count, 10);
+}
diff --git a/test/hotspot/gtest/gc/x/test_xBitField.cpp b/test/hotspot/gtest/gc/x/test_xBitField.cpp
new file mode 100644
index 00000000000..248322b2a07
--- /dev/null
+++ b/test/hotspot/gtest/gc/x/test_xBitField.cpp
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xBitField.hpp"
+#include "unittest.hpp"
+
+TEST(XBitFieldTest, test) {
+  typedef XBitField<uint64_t, bool,      0,  1>    field_bool;
+  typedef XBitField<uint64_t, uint8_t,   1,  8>    field_uint8;
+  typedef XBitField<uint64_t, uint16_t,  2, 16>    field_uint16;
+  typedef XBitField<uint64_t, uint32_t, 32, 32>    field_uint32;
+  typedef XBitField<uint64_t, uint64_t,  0, 63>    field_uint64;
+  typedef XBitField<uint64_t, void*,     1, 61, 3> field_pointer;
+
+  uint64_t entry;
+
+  {
+    const bool value = false;
+    entry = field_bool::encode(value);
+    EXPECT_EQ(field_bool::decode(entry), value) << "Should be equal";
+  }
+
+  {
+    const bool value = true;
+    entry = field_bool::encode(value);
+      EXPECT_EQ(field_bool::decode(entry), value) << "Should be equal";
+  }
+
+  {
+    const uint8_t value = ~(uint8_t)0;
+    entry = field_uint8::encode(value);
+    EXPECT_EQ(field_uint8::decode(entry), value) << "Should be equal";
+  }
+
+  {
+    const uint16_t value = ~(uint16_t)0;
+    entry = field_uint16::encode(value);
+    EXPECT_EQ(field_uint16::decode(entry), value) << "Should be equal";
+  }
+
+  {
+    const uint32_t value = ~(uint32_t)0;
+    entry = field_uint32::encode(value);
+    EXPECT_EQ(field_uint32::decode(entry), value) << "Should be equal";
+  }
+
+  {
+    const uint64_t value = ~(uint64_t)0 >> 1;
+    entry = field_uint64::encode(value);
+    EXPECT_EQ(field_uint64::decode(entry), value) << "Should be equal";
+  }
+
+  {
+    void* const value = (void*)(~(uintptr_t)0 << 3);
+    entry = field_pointer::encode(value);
+    EXPECT_EQ(field_pointer::decode(entry), value) << "Should be equal";
+  }
+}
diff --git a/test/hotspot/gtest/gc/x/test_xBitMap.cpp b/test/hotspot/gtest/gc/x/test_xBitMap.cpp
new file mode 100644
index 00000000000..2d3cb09c7ed
--- /dev/null
+++ b/test/hotspot/gtest/gc/x/test_xBitMap.cpp
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xBitMap.inline.hpp"
+#include "unittest.hpp"
+
+class XBitMapTest : public ::testing::Test {
+protected:
+  static void test_set_pair_unset(size_t size, bool finalizable) {
+    XBitMap bitmap(size);
+
+    for (BitMap::idx_t i = 0; i < size - 1; i++) {
+      if ((i + 1) % BitsPerWord == 0) {
+        // Can't set pairs of bits in different words.
+        continue;
+      }
+
+      // XBitMaps are not cleared when constructed.
+      bitmap.clear();
+
+      bool inc_live = false;
+
+      bool ret = bitmap.par_set_bit_pair(i, finalizable, inc_live);
+      EXPECT_TRUE(ret) << "Failed to set bit";
+      EXPECT_TRUE(inc_live) << "Should have set inc_live";
+
+      // First bit should always be set
+      EXPECT_TRUE(bitmap.at(i)) << "Should be set";
+
+      // Second bit should only be set when marking strong
+      EXPECT_NE(bitmap.at(i + 1), finalizable);
+    }
+  }
+
+  static void test_set_pair_set(size_t size, bool finalizable) {
+    XBitMap bitmap(size);
+
+    for (BitMap::idx_t i = 0; i < size - 1; i++) {
+      if ((i + 1) % BitsPerWord == 0) {
+        // Can't set pairs of bits in different words.
+        continue;
+      }
+
+      // Fill the bitmap with ones.
+      bitmap.set_range(0, size);
+
+      bool inc_live = false;
+
+      bool ret = bitmap.par_set_bit_pair(i, finalizable, inc_live);
+      EXPECT_FALSE(ret) << "Should not succeed setting bit";
+      EXPECT_FALSE(inc_live) << "Should not have set inc_live";
+
+      // Both bits were pre-set.
+      EXPECT_TRUE(bitmap.at(i)) << "Should be set";
+      EXPECT_TRUE(bitmap.at(i + 1)) << "Should be set";
+    }
+  }
+
+  static void test_set_pair_set(bool finalizable) {
+    test_set_pair_set(2,   finalizable);
+    test_set_pair_set(62,  finalizable);
+    test_set_pair_set(64,  finalizable);
+    test_set_pair_set(66,  finalizable);
+    test_set_pair_set(126, finalizable);
+    test_set_pair_set(128, finalizable);
+  }
+
+  static void test_set_pair_unset(bool finalizable) {
+    test_set_pair_unset(2,   finalizable);
+    test_set_pair_unset(62,  finalizable);
+    test_set_pair_unset(64,  finalizable);
+    test_set_pair_unset(66,  finalizable);
+    test_set_pair_unset(126, finalizable);
+    test_set_pair_unset(128, finalizable);
+  }
+
+};
+
+TEST_F(XBitMapTest, test_set_pair_set) {
+  test_set_pair_set(false);
+  test_set_pair_set(true);
+}
+
+TEST_F(XBitMapTest, test_set_pair_unset) {
+  test_set_pair_unset(false);
+  test_set_pair_unset(true);
+}
diff --git a/test/hotspot/gtest/gc/x/test_xForwarding.cpp b/test/hotspot/gtest/gc/x/test_xForwarding.cpp
new file mode 100644
index 00000000000..de850304ebb
--- /dev/null
+++ b/test/hotspot/gtest/gc/x/test_xForwarding.cpp
@@ -0,0 +1,205 @@
+/*
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xAddress.inline.hpp"
+#include "gc/x/xForwarding.inline.hpp"
+#include "gc/x/xForwardingAllocator.inline.hpp"
+#include "gc/x/xGlobals.hpp"
+#include "gc/x/xPage.inline.hpp"
+#include "unittest.hpp"
+
+using namespace testing;
+
+#define CAPTURE_DELIM "\n"
+#define CAPTURE1(expression) #expression << " evaluates to " << expression
+#define CAPTURE2(e0, e1)                 CAPTURE1(e0) << CAPTURE_DELIM << CAPTURE1(e1)
+
+#define CAPTURE(expression) CAPTURE1(expression)
+
+class XForwardingTest : public Test {
+public:
+  // Helper functions
+
+  class SequenceToFromIndex : AllStatic {
+  public:
+    static uintptr_t even(size_t sequence_number) {
+      return sequence_number * 2;
+    }
+    static uintptr_t odd(size_t sequence_number) {
+      return even(sequence_number) + 1;
+    }
+    static uintptr_t one_to_one(size_t sequence_number) {
+      return sequence_number;
+    }
+  };
+
+  // Test functions
+
+  static void setup(XForwarding* forwarding) {
+    EXPECT_PRED1(is_power_of_2<size_t>, forwarding->_entries.length()) << CAPTURE(forwarding->_entries.length());
+  }
+
+  static void find_empty(XForwarding* forwarding) {
+    size_t size = forwarding->_entries.length();
+    size_t entries_to_check = size * 2;
+
+    for (size_t i = 0; i < entries_to_check; i++) {
+      uintptr_t from_index = SequenceToFromIndex::one_to_one(i);
+
+      XForwardingCursor cursor;
+      XForwardingEntry entry = forwarding->find(from_index, &cursor);
+      EXPECT_FALSE(entry.populated()) << CAPTURE2(from_index, size);
+    }
+  }
+
+  static void find_full(XForwarding* forwarding) {
+    size_t size = forwarding->_entries.length();
+    size_t entries_to_populate = size;
+
+    // Populate
+    for (size_t i = 0; i < entries_to_populate; i++) {
+      uintptr_t from_index = SequenceToFromIndex::one_to_one(i);
+
+      XForwardingCursor cursor;
+      XForwardingEntry entry = forwarding->find(from_index, &cursor);
+      ASSERT_FALSE(entry.populated()) << CAPTURE2(from_index, size);
+
+      forwarding->insert(from_index, from_index, &cursor);
+    }
+
+    // Verify
+    for (size_t i = 0; i < entries_to_populate; i++) {
+      uintptr_t from_index = SequenceToFromIndex::one_to_one(i);
+
+      XForwardingCursor cursor;
+      XForwardingEntry entry = forwarding->find(from_index, &cursor);
+      ASSERT_TRUE(entry.populated()) << CAPTURE2(from_index, size);
+
+      ASSERT_EQ(entry.from_index(), from_index) << CAPTURE(size);
+      ASSERT_EQ(entry.to_offset(), from_index) << CAPTURE(size);
+    }
+  }
+
+  static void find_every_other(XForwarding* forwarding) {
+    size_t size = forwarding->_entries.length();
+    size_t entries_to_populate = size / 2;
+
+    // Populate even from indices
+    for (size_t i = 0; i < entries_to_populate; i++) {
+      uintptr_t from_index = SequenceToFromIndex::even(i);
+
+      XForwardingCursor cursor;
+      XForwardingEntry entry = forwarding->find(from_index, &cursor);
+      ASSERT_FALSE(entry.populated()) << CAPTURE2(from_index, size);
+
+      forwarding->insert(from_index, from_index, &cursor);
+    }
+
+    // Verify populated even indices
+    for (size_t i = 0; i < entries_to_populate; i++) {
+      uintptr_t from_index = SequenceToFromIndex::even(i);
+
+      XForwardingCursor cursor;
+      XForwardingEntry entry = forwarding->find(from_index, &cursor);
+      ASSERT_TRUE(entry.populated()) << CAPTURE2(from_index, size);
+
+      ASSERT_EQ(entry.from_index(), from_index) << CAPTURE(size);
+      ASSERT_EQ(entry.to_offset(), from_index) << CAPTURE(size);
+    }
+
+    // Verify empty odd indices
+    //
+    // This check could be done on a larger range of sequence numbers,
+    // but currently entries_to_populate is used.
+    for (size_t i = 0; i < entries_to_populate; i++) {
+      uintptr_t from_index = SequenceToFromIndex::odd(i);
+
+      XForwardingCursor cursor;
+      XForwardingEntry entry = forwarding->find(from_index, &cursor);
+
+      ASSERT_FALSE(entry.populated()) << CAPTURE2(from_index, size);
+    }
+  }
+
+  static void test(void (*function)(XForwarding*), uint32_t size) {
+    // Create page
+    const XVirtualMemory vmem(0, XPageSizeSmall);
+    const XPhysicalMemory pmem(XPhysicalMemorySegment(0, XPageSizeSmall, true));
+    XPage page(XPageTypeSmall, vmem, pmem);
+
+    page.reset();
+
+    const size_t object_size = 16;
+    const uintptr_t object = page.alloc_object(object_size);
+
+    XGlobalSeqNum++;
+
+    bool dummy = false;
+    page.mark_object(XAddress::marked(object), dummy, dummy);
+
+    const uint32_t live_objects = size;
+    const size_t live_bytes = live_objects * object_size;
+    page.inc_live(live_objects, live_bytes);
+
+    // Setup allocator
+    XForwardingAllocator allocator;
+    const uint32_t nentries = XForwarding::nentries(&page);
+    allocator.reset((sizeof(XForwarding)) + (nentries * sizeof(XForwardingEntry)));
+
+    // Setup forwarding
+    XForwarding* const forwarding = XForwarding::alloc(&allocator, &page);
+
+    // Actual test function
+    (*function)(forwarding);
+  }
+
+  // Run the given function with a few different input values.
+  static void test(void (*function)(XForwarding*)) {
+    test(function, 1);
+    test(function, 2);
+    test(function, 3);
+    test(function, 4);
+    test(function, 7);
+    test(function, 8);
+    test(function, 1023);
+    test(function, 1024);
+    test(function, 1025);
+  }
+};
+
+TEST_F(XForwardingTest, setup) {
+  test(&XForwardingTest::setup);
+}
+
+TEST_F(XForwardingTest, find_empty) {
+  test(&XForwardingTest::find_empty);
+}
+
+TEST_F(XForwardingTest, find_full) {
+  test(&XForwardingTest::find_full);
+}
+
+TEST_F(XForwardingTest, find_every_other) {
+  test(&XForwardingTest::find_every_other);
+}
diff --git a/test/hotspot/gtest/gc/x/test_xList.cpp b/test/hotspot/gtest/gc/x/test_xList.cpp
new file mode 100644
index 00000000000..f4766ce99e2
--- /dev/null
+++ b/test/hotspot/gtest/gc/x/test_xList.cpp
@@ -0,0 +1,155 @@
+/*
+ * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xList.inline.hpp"
+#include "unittest.hpp"
+
+#ifndef PRODUCT
+
+class XTestEntry {
+  friend class XList<XTestEntry>;
+
+private:
+  const int             _id;
+  XListNode<XTestEntry> _node;
+
+public:
+  XTestEntry(int id) :
+      _id(id),
+      _node() {}
+
+  int id() const {
+    return _id;
+  }
+};
+
+class XListTest : public ::testing::Test {
+protected:
+  static void assert_sorted(XList<XTestEntry>* list) {
+    // Iterate forward
+    {
+      int count = list->first()->id();
+      XListIterator<XTestEntry> iter(list);
+      for (XTestEntry* entry; iter.next(&entry);) {
+        ASSERT_EQ(entry->id(), count);
+        count++;
+      }
+    }
+
+    // Iterate backward
+    {
+      int count = list->last()->id();
+      XListReverseIterator<XTestEntry> iter(list);
+      for (XTestEntry* entry; iter.next(&entry);) {
+        EXPECT_EQ(entry->id(), count);
+        count--;
+      }
+    }
+  }
+};
+
+TEST_F(XListTest, test_insert) {
+  XList<XTestEntry> list;
+  XTestEntry e0(0);
+  XTestEntry e1(1);
+  XTestEntry e2(2);
+  XTestEntry e3(3);
+  XTestEntry e4(4);
+  XTestEntry e5(5);
+
+  list.insert_first(&e2);
+  list.insert_before(&e2, &e1);
+  list.insert_after(&e2, &e3);
+  list.insert_last(&e4);
+  list.insert_first(&e0);
+  list.insert_last(&e5);
+
+  EXPECT_EQ(list.size(), 6u);
+  assert_sorted(&list);
+
+  for (int i = 0; i < 6; i++) {
+    XTestEntry* e = list.remove_first();
+    EXPECT_EQ(e->id(), i);
+  }
+
+  EXPECT_EQ(list.size(), 0u);
+}
+
+TEST_F(XListTest, test_remove) {
+  // Remove first
+  {
+    XList<XTestEntry> list;
+    XTestEntry e0(0);
+    XTestEntry e1(1);
+    XTestEntry e2(2);
+    XTestEntry e3(3);
+    XTestEntry e4(4);
+    XTestEntry e5(5);
+
+    list.insert_last(&e0);
+    list.insert_last(&e1);
+    list.insert_last(&e2);
+    list.insert_last(&e3);
+    list.insert_last(&e4);
+    list.insert_last(&e5);
+
+    EXPECT_EQ(list.size(), 6u);
+
+    for (int i = 0; i < 6; i++) {
+      XTestEntry* e = list.remove_first();
+      EXPECT_EQ(e->id(), i);
+    }
+
+    EXPECT_EQ(list.size(), 0u);
+  }
+
+  // Remove last
+  {
+    XList<XTestEntry> list;
+    XTestEntry e0(0);
+    XTestEntry e1(1);
+    XTestEntry e2(2);
+    XTestEntry e3(3);
+    XTestEntry e4(4);
+    XTestEntry e5(5);
+
+    list.insert_last(&e0);
+    list.insert_last(&e1);
+    list.insert_last(&e2);
+    list.insert_last(&e3);
+    list.insert_last(&e4);
+    list.insert_last(&e5);
+
+    EXPECT_EQ(list.size(), 6u);
+
+    for (int i = 5; i >= 0; i--) {
+      XTestEntry* e = list.remove_last();
+      EXPECT_EQ(e->id(), i);
+    }
+
+    EXPECT_EQ(list.size(), 0u);
+  }
+}
+
+#endif // PRODUCT
diff --git a/test/hotspot/gtest/gc/x/test_xLiveMap.cpp b/test/hotspot/gtest/gc/x/test_xLiveMap.cpp
new file mode 100644
index 00000000000..d57790e9dab
--- /dev/null
+++ b/test/hotspot/gtest/gc/x/test_xLiveMap.cpp
@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xLiveMap.inline.hpp"
+#include "unittest.hpp"
+
+class XLiveMapTest : public ::testing::Test {
+protected:
+  static void strongly_live_for_large_xpage() {
+    // Large XPages only have room for one object.
+    XLiveMap livemap(1);
+
+    bool inc_live;
+    uintptr_t object = 0u;
+
+    // Mark the object strong.
+    livemap.set(object, false /* finalizable */, inc_live);
+
+    // Check that both bits are in the same segment.
+    ASSERT_EQ(livemap.index_to_segment(0), livemap.index_to_segment(1));
+
+    // Check that the object was marked.
+    ASSERT_TRUE(livemap.get(0));
+
+    // Check that the object was strongly marked.
+    ASSERT_TRUE(livemap.get(1));
+
+    ASSERT_TRUE(inc_live);
+  }
+};
+
+TEST_F(XLiveMapTest, strongly_live_for_large_xpage) {
+  strongly_live_for_large_xpage();
+}
diff --git a/test/hotspot/gtest/gc/x/test_xPhysicalMemory.cpp b/test/hotspot/gtest/gc/x/test_xPhysicalMemory.cpp
new file mode 100644
index 00000000000..f22032632e9
--- /dev/null
+++ b/test/hotspot/gtest/gc/x/test_xPhysicalMemory.cpp
@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xPhysicalMemory.inline.hpp"
+#include "unittest.hpp"
+
+TEST(XPhysicalMemoryTest, copy) {
+  const XPhysicalMemorySegment seg0(0, 100, true);
+  const XPhysicalMemorySegment seg1(200, 100, true);
+
+  XPhysicalMemory pmem0;
+  pmem0.add_segment(seg0);
+  EXPECT_EQ(pmem0.nsegments(), 1);
+  EXPECT_EQ(pmem0.segment(0).size(), 100u);
+
+  XPhysicalMemory pmem1;
+  pmem1.add_segment(seg0);
+  pmem1.add_segment(seg1);
+  EXPECT_EQ(pmem1.nsegments(), 2);
+  EXPECT_EQ(pmem1.segment(0).size(), 100u);
+  EXPECT_EQ(pmem1.segment(1).size(), 100u);
+
+  XPhysicalMemory pmem2(pmem0);
+  EXPECT_EQ(pmem2.nsegments(), 1);
+  EXPECT_EQ(pmem2.segment(0).size(), 100u);
+
+  pmem2 = pmem1;
+  EXPECT_EQ(pmem2.nsegments(), 2);
+  EXPECT_EQ(pmem2.segment(0).size(), 100u);
+  EXPECT_EQ(pmem2.segment(1).size(), 100u);
+}
+
+TEST(XPhysicalMemoryTest, add) {
+  const XPhysicalMemorySegment seg0(0, 1, true);
+  const XPhysicalMemorySegment seg1(1, 1, true);
+  const XPhysicalMemorySegment seg2(2, 1, true);
+  const XPhysicalMemorySegment seg3(3, 1, true);
+  const XPhysicalMemorySegment seg4(4, 1, true);
+  const XPhysicalMemorySegment seg5(5, 1, true);
+  const XPhysicalMemorySegment seg6(6, 1, true);
+
+  XPhysicalMemory pmem0;
+  EXPECT_EQ(pmem0.nsegments(), 0);
+  EXPECT_EQ(pmem0.is_null(), true);
+
+  XPhysicalMemory pmem1;
+  pmem1.add_segment(seg0);
+  pmem1.add_segment(seg1);
+  pmem1.add_segment(seg2);
+  pmem1.add_segment(seg3);
+  pmem1.add_segment(seg4);
+  pmem1.add_segment(seg5);
+  pmem1.add_segment(seg6);
+  EXPECT_EQ(pmem1.nsegments(), 1);
+  EXPECT_EQ(pmem1.segment(0).size(), 7u);
+  EXPECT_EQ(pmem1.is_null(), false);
+
+  XPhysicalMemory pmem2;
+  pmem2.add_segment(seg0);
+  pmem2.add_segment(seg1);
+  pmem2.add_segment(seg2);
+  pmem2.add_segment(seg4);
+  pmem2.add_segment(seg5);
+  pmem2.add_segment(seg6);
+  EXPECT_EQ(pmem2.nsegments(), 2);
+  EXPECT_EQ(pmem2.segment(0).size(), 3u);
+  EXPECT_EQ(pmem2.segment(1).size(), 3u);
+  EXPECT_EQ(pmem2.is_null(), false);
+
+  XPhysicalMemory pmem3;
+  pmem3.add_segment(seg0);
+  pmem3.add_segment(seg2);
+  pmem3.add_segment(seg3);
+  pmem3.add_segment(seg4);
+  pmem3.add_segment(seg6);
+  EXPECT_EQ(pmem3.nsegments(), 3);
+  EXPECT_EQ(pmem3.segment(0).size(), 1u);
+  EXPECT_EQ(pmem3.segment(1).size(), 3u);
+  EXPECT_EQ(pmem3.segment(2).size(), 1u);
+  EXPECT_EQ(pmem3.is_null(), false);
+
+  XPhysicalMemory pmem4;
+  pmem4.add_segment(seg0);
+  pmem4.add_segment(seg2);
+  pmem4.add_segment(seg4);
+  pmem4.add_segment(seg6);
+  EXPECT_EQ(pmem4.nsegments(), 4);
+  EXPECT_EQ(pmem4.segment(0).size(), 1u);
+  EXPECT_EQ(pmem4.segment(1).size(), 1u);
+  EXPECT_EQ(pmem4.segment(2).size(), 1u);
+  EXPECT_EQ(pmem4.segment(3).size(), 1u);
+  EXPECT_EQ(pmem4.is_null(), false);
+}
+
+TEST(XPhysicalMemoryTest, remove) {
+  XPhysicalMemory pmem;
+
+  pmem.add_segment(XPhysicalMemorySegment(10, 10, true));
+  pmem.add_segment(XPhysicalMemorySegment(30, 10, true));
+  pmem.add_segment(XPhysicalMemorySegment(50, 10, true));
+  EXPECT_EQ(pmem.nsegments(), 3);
+  EXPECT_EQ(pmem.size(), 30u);
+  EXPECT_FALSE(pmem.is_null());
+
+  pmem.remove_segments();
+  EXPECT_EQ(pmem.nsegments(), 0);
+  EXPECT_EQ(pmem.size(), 0u);
+  EXPECT_TRUE(pmem.is_null());
+}
+
+TEST(XPhysicalMemoryTest, split) {
+  XPhysicalMemory pmem;
+
+  pmem.add_segment(XPhysicalMemorySegment(0, 10, true));
+  pmem.add_segment(XPhysicalMemorySegment(10, 10, true));
+  pmem.add_segment(XPhysicalMemorySegment(30, 10, true));
+  EXPECT_EQ(pmem.nsegments(), 2);
+  EXPECT_EQ(pmem.size(), 30u);
+
+  XPhysicalMemory pmem0 = pmem.split(1);
+  EXPECT_EQ(pmem0.nsegments(), 1);
+  EXPECT_EQ(pmem0.size(), 1u);
+  EXPECT_EQ(pmem.nsegments(), 2);
+  EXPECT_EQ(pmem.size(), 29u);
+
+  XPhysicalMemory pmem1 = pmem.split(25);
+  EXPECT_EQ(pmem1.nsegments(), 2);
+  EXPECT_EQ(pmem1.size(), 25u);
+  EXPECT_EQ(pmem.nsegments(), 1);
+  EXPECT_EQ(pmem.size(), 4u);
+
+  XPhysicalMemory pmem2 = pmem.split(4);
+  EXPECT_EQ(pmem2.nsegments(), 1);
+  EXPECT_EQ(pmem2.size(), 4u);
+  EXPECT_EQ(pmem.nsegments(), 0);
+  EXPECT_EQ(pmem.size(), 0u);
+}
+
+TEST(XPhysicalMemoryTest, split_committed) {
+  XPhysicalMemory pmem0;
+  pmem0.add_segment(XPhysicalMemorySegment(0, 10, true));
+  pmem0.add_segment(XPhysicalMemorySegment(10, 10, false));
+  pmem0.add_segment(XPhysicalMemorySegment(20, 10, true));
+  pmem0.add_segment(XPhysicalMemorySegment(30, 10, false));
+  EXPECT_EQ(pmem0.nsegments(), 4);
+  EXPECT_EQ(pmem0.size(), 40u);
+
+  XPhysicalMemory pmem1 = pmem0.split_committed();
+  EXPECT_EQ(pmem0.nsegments(), 2);
+  EXPECT_EQ(pmem0.size(), 20u);
+  EXPECT_EQ(pmem1.nsegments(), 2);
+  EXPECT_EQ(pmem1.size(), 20u);
+}
diff --git a/test/hotspot/gtest/gc/x/test_xVirtualMemory.cpp b/test/hotspot/gtest/gc/x/test_xVirtualMemory.cpp
new file mode 100644
index 00000000000..6698ccfa045
--- /dev/null
+++ b/test/hotspot/gtest/gc/x/test_xVirtualMemory.cpp
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/x/xVirtualMemory.inline.hpp"
+#include "unittest.hpp"
+
+TEST(XVirtualMemory, split) {
+  XVirtualMemory vmem(0, 10);
+
+  XVirtualMemory vmem0 = vmem.split(0);
+  EXPECT_EQ(vmem0.size(), 0u);
+  EXPECT_EQ(vmem.size(), 10u);
+
+  XVirtualMemory vmem1 = vmem.split(5);
+  EXPECT_EQ(vmem1.size(), 5u);
+  EXPECT_EQ(vmem.size(), 5u);
+
+  XVirtualMemory vmem2 = vmem.split(5);
+  EXPECT_EQ(vmem2.size(), 5u);
+  EXPECT_EQ(vmem.size(), 0u);
+
+  XVirtualMemory vmem3 = vmem.split(0);
+  EXPECT_EQ(vmem3.size(), 0u);
+}
diff --git a/test/hotspot/gtest/gc/z/test_zAddress.cpp b/test/hotspot/gtest/gc/z/test_zAddress.cpp
index bf5344c8da4..cbfec6747b4 100644
--- a/test/hotspot/gtest/gc/z/test_zAddress.cpp
+++ b/test/hotspot/gtest/gc/z/test_zAddress.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,108 +28,410 @@
 
 class ZAddressTest : public ::testing::Test {
 protected:
-  static void is_good_bit(uintptr_t bit_mask) {
-    // Setup
-    ZAddress::initialize();
-    ZAddress::set_good_mask(bit_mask);
-
-    // Test that a pointer with only the given bit is considered good.
-    EXPECT_EQ(ZAddress::is_good(ZAddressMetadataMarked0),  (bit_mask == ZAddressMetadataMarked0));
-    EXPECT_EQ(ZAddress::is_good(ZAddressMetadataMarked1),  (bit_mask == ZAddressMetadataMarked1));
-    EXPECT_EQ(ZAddress::is_good(ZAddressMetadataRemapped), (bit_mask == ZAddressMetadataRemapped));
-
-    // Test that a pointer with the given bit and some extra bits is considered good.
-    EXPECT_EQ(ZAddress::is_good(ZAddressMetadataMarked0  | 0x8),(bit_mask == ZAddressMetadataMarked0));
-    EXPECT_EQ(ZAddress::is_good(ZAddressMetadataMarked1  | 0x8), (bit_mask == ZAddressMetadataMarked1));
-    EXPECT_EQ(ZAddress::is_good(ZAddressMetadataRemapped | 0x8), (bit_mask == ZAddressMetadataRemapped));
-
-    // Test that null is not considered good.
-    EXPECT_FALSE(ZAddress::is_good(0));
+  static zpointer color(uintptr_t value, uintptr_t color) {
+    return ZAddress::color(zaddress(value | ZAddressHeapBase), color);
   }
 
-  static void is_good_or_null_bit(uintptr_t bit_mask) {
-    // Setup
-    ZAddress::initialize();
-    ZAddress::set_good_mask(bit_mask);
+  static const uintptr_t valid_value = (1 << 3 /* LogMinObjectAlignment */);
+  static const uintptr_t null_value = 0;
 
-    // Test that a pointer with only the given bit is considered good.
-    EXPECT_EQ(ZAddress::is_good_or_null(ZAddressMetadataMarked0),  (bit_mask == ZAddressMetadataMarked0));
-    EXPECT_EQ(ZAddress::is_good_or_null(ZAddressMetadataMarked1),  (bit_mask == ZAddressMetadataMarked1));
-    EXPECT_EQ(ZAddress::is_good_or_null(ZAddressMetadataRemapped), (bit_mask == ZAddressMetadataRemapped));
+  enum ZColor {
+    Uncolored,
+    RemappedYoung0,
+    RemappedYoung1,
+    RemappedOld0,
+    RemappedOld1,
+    MarkedYoung0,
+    MarkedYoung1,
+    MarkedOld0,
+    MarkedOld1,
+    Finalizable0,
+    Finalizable1,
+    Remembered0,
+    Remembered1,
+    Remembered11
+  };
 
-    // Test that a pointer with the given bit and some extra bits is considered good.
-    EXPECT_EQ(ZAddress::is_good_or_null(ZAddressMetadataMarked0  | 0x8), (bit_mask == ZAddressMetadataMarked0));
-    EXPECT_EQ(ZAddress::is_good_or_null(ZAddressMetadataMarked1  | 0x8), (bit_mask == ZAddressMetadataMarked1));
-    EXPECT_EQ(ZAddress::is_good_or_null(ZAddressMetadataRemapped | 0x8), (bit_mask == ZAddressMetadataRemapped));
+  static uintptr_t make_color(ZColor remembered, ZColor remapped_young, ZColor remapped_old, ZColor marked_young, ZColor marked_old) {
+    uintptr_t color = 0;
+    switch (remapped_young) {
+    case RemappedYoung0: {
+      switch (remapped_old) {
+      case RemappedOld0:
+        color |= ZPointer::remap_bits(ZPointerRemapped00);
+        break;
+      case RemappedOld1:
+        color |= ZPointer::remap_bits(ZPointerRemapped10);
+        break;
+      default:
+        EXPECT_TRUE(false);
+      }
+      break;
+    }
+    case RemappedYoung1: {
+      switch (remapped_old) {
+      case RemappedOld0:
+        color |= ZPointer::remap_bits(ZPointerRemapped01);
+        break;
+      case RemappedOld1:
+        color |= ZPointer::remap_bits(ZPointerRemapped11);
+        break;
+      default:
+        EXPECT_TRUE(false);
+      }
+      break;
+    }
+    default:
+      EXPECT_TRUE(false);
+    }
 
-    // Test that null is considered good_or_null.
-    EXPECT_TRUE(ZAddress::is_good_or_null(0));
+    switch (marked_young) {
+    case MarkedYoung0:
+      color |= ZPointerMarkedYoung0;
+      break;
+    case MarkedYoung1:
+      color |= ZPointerMarkedYoung1;
+      break;
+    default:
+      EXPECT_TRUE(false);
+    }
+
+    switch (marked_old) {
+    case MarkedOld0:
+      color |= ZPointerMarkedOld0;
+      break;
+    case MarkedOld1:
+      color |= ZPointerMarkedOld1;
+      break;
+    case Finalizable0:
+      color |= ZPointerFinalizable0;
+      break;
+    case Finalizable1:
+      color |= ZPointerFinalizable1;
+      break;
+    default:
+      EXPECT_TRUE(false);
+    }
+
+    switch (remembered) {
+    case Remembered0:
+      color |= ZPointerRemembered0;
+      break;
+    case Remembered1:
+      color |= ZPointerRemembered1;
+      break;
+    case Remembered11:
+      color |= ZPointerRemembered0 | ZPointerRemembered1;
+      break;
+    default:
+      EXPECT_TRUE(false);
+    }
+
+    return color;
   }
 
-  static void finalizable() {
+  static zpointer color(uintptr_t addr,
+                        ZColor remembered,
+                        ZColor remapped_young,
+                        ZColor remapped_old,
+                        ZColor marked_young,
+                        ZColor marked_old) {
+    if (remembered == Uncolored &&
+        remapped_young == Uncolored &&
+        remapped_old == Uncolored &&
+        marked_young == Uncolored &&
+        marked_old == Uncolored) {
+      return zpointer(addr);
+    } else {
+      return color(addr, make_color(remembered, remapped_young, remapped_old, marked_young, marked_old));
+    }
+  }
+
+  static bool is_remapped_young_odd(uintptr_t bits) {
+    return ZPointer::remap_bits(bits) & (ZPointerRemapped01 | ZPointerRemapped11);
+  }
+
+  static bool is_remapped_old_odd(uintptr_t bits) {
+    return ZPointer::remap_bits(bits) & (ZPointerRemapped10 | ZPointerRemapped11);
+  }
+
+  static bool is_marked_young_odd(uintptr_t bits) {
+    return bits & ZPointerMarkedYoung1;
+  }
+
+  static bool is_marked_old_odd(uintptr_t bits) {
+    return bits & (ZPointerMarkedOld1 | ZPointerFinalizable1);
+  }
+
+  static bool is_remembered(uintptr_t bits) {
+    return bits & (ZPointerRemembered0 | ZPointerRemembered1);
+  }
+
+  static bool is_remembered_odd(uintptr_t bits) {
+    return bits & (ZPointerRemembered1);
+  }
+
+  static bool is_remembered_even(uintptr_t bits) {
+    return bits & (ZPointerRemembered0);
+  }
+
+  static void test_is_checks_on(uintptr_t value,
+                                ZColor remembered,
+                                ZColor remapped_young,
+                                ZColor remapped_old,
+                                ZColor marked_young,
+                                ZColor marked_old) {
+    const zpointer ptr = color(value, remembered, remapped_young, remapped_old, marked_young, marked_old);
+    uintptr_t ptr_raw = untype(ptr);
+
+    EXPECT_TRUE(ZPointerLoadGoodMask != 0);
+    EXPECT_TRUE(ZPointerStoreGoodMask != 0);
+
+    bool ptr_raw_null = ptr_raw == 0;
+    bool global_remapped_old_odd = is_remapped_old_odd(ZPointerLoadGoodMask);
+    bool global_remapped_young_odd = is_remapped_young_odd(ZPointerLoadGoodMask);
+    bool global_marked_old_odd = is_marked_old_odd(ZPointerStoreGoodMask);
+    bool global_marked_young_odd = is_marked_young_odd(ZPointerStoreGoodMask);
+    bool global_remembered_odd = is_remembered_odd(ZPointerStoreGoodMask);
+    bool global_remembered_even = is_remembered_even(ZPointerStoreGoodMask);
+
+    if (ptr_raw_null) {
+      EXPECT_FALSE(ZPointer::is_marked_any_old(ptr));
+      EXPECT_FALSE(ZPointer::is_load_good(ptr));
+      EXPECT_TRUE(ZPointer::is_load_good_or_null(ptr));
+      EXPECT_FALSE(ZPointer::is_load_bad(ptr));
+      EXPECT_FALSE(ZPointer::is_mark_good(ptr));
+      EXPECT_TRUE(ZPointer::is_mark_good_or_null(ptr));
+      EXPECT_FALSE(ZPointer::is_mark_bad(ptr));
+      EXPECT_FALSE(ZPointer::is_store_good(ptr));
+      EXPECT_TRUE(ZPointer::is_store_good_or_null(ptr));
+      EXPECT_FALSE(ZPointer::is_store_bad(ptr));
+    } else {
+      bool ptr_remapped_old_odd = is_remapped_old_odd(ptr_raw);
+      bool ptr_remapped_young_odd = is_remapped_young_odd(ptr_raw);
+      bool ptr_marked_old_odd = is_marked_old_odd(ptr_raw);
+      bool ptr_marked_young_odd = is_marked_young_odd(ptr_raw);
+      bool ptr_final = ptr_raw & (ZPointerFinalizable0 | ZPointerFinalizable1);
+      bool ptr_remembered = is_power_of_2(ptr_raw & (ZPointerRemembered0 | ZPointerRemembered1));
+      bool ptr_remembered_odd = is_remembered_odd(ptr_raw);
+      bool ptr_remembered_even = is_remembered_even(ptr_raw);
+      bool ptr_colored_null = !ptr_raw_null && (ptr_raw & ~ZPointerAllMetadataMask) == 0;
+
+      bool same_old_marking = global_marked_old_odd == ptr_marked_old_odd;
+      bool same_young_marking = global_marked_young_odd == ptr_marked_young_odd;
+      bool same_old_remapping = global_remapped_old_odd == ptr_remapped_old_odd;
+      bool same_young_remapping = global_remapped_young_odd == ptr_remapped_young_odd;
+      bool same_remembered = ptr_remembered_even == global_remembered_even && ptr_remembered_odd == global_remembered_odd;
+
+      EXPECT_EQ(ZPointer::is_marked_finalizable(ptr), same_old_marking && ptr_final);
+      EXPECT_EQ(ZPointer::is_marked_any_old(ptr), same_old_marking);
+      EXPECT_EQ(ZPointer::is_remapped(ptr), same_old_remapping && same_young_remapping);
+      EXPECT_EQ(ZPointer::is_load_good(ptr), same_old_remapping && same_young_remapping);
+      EXPECT_EQ(ZPointer::is_load_good_or_null(ptr), same_old_remapping && same_young_remapping);
+      EXPECT_EQ(ZPointer::is_load_bad(ptr), !same_old_remapping || !same_young_remapping);
+      EXPECT_EQ(ZPointer::is_mark_good(ptr), same_young_remapping && same_old_remapping && same_young_marking && same_old_marking);
+      EXPECT_EQ(ZPointer::is_mark_good_or_null(ptr), same_young_remapping && same_old_remapping && same_young_marking && same_old_marking);
+      EXPECT_EQ(ZPointer::is_mark_bad(ptr), !same_young_remapping || !same_old_remapping || !same_young_marking || !same_old_marking);
+      EXPECT_EQ(ZPointer::is_store_good(ptr), same_young_remapping && same_old_remapping && same_young_marking && same_old_marking && ptr_remembered && same_remembered);
+      EXPECT_EQ(ZPointer::is_store_good_or_null(ptr), same_young_remapping && same_old_remapping && same_young_marking && same_old_marking && ptr_remembered && same_remembered);
+      EXPECT_EQ(ZPointer::is_store_bad(ptr), !same_young_remapping || !same_old_remapping || !same_young_marking || !same_old_marking || !ptr_remembered || !same_remembered);
+    }
+  }
+
+  static void test_is_checks_on_all() {
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung0, RemappedOld0, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung0, RemappedOld0, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung0, RemappedOld0, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung0, RemappedOld0, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung0, RemappedOld0, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung0, RemappedOld0, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung0, RemappedOld0, MarkedYoung1, MarkedOld1);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung0, RemappedOld0, MarkedYoung1, MarkedOld1);
+
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung0, RemappedOld1, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung0, RemappedOld1, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung0, RemappedOld1, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung0, RemappedOld1, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung0, RemappedOld1, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung0, RemappedOld1, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung0, RemappedOld1, MarkedYoung1, MarkedOld1);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung0, RemappedOld1, MarkedYoung1, MarkedOld1);
+
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung1, RemappedOld0, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung1, RemappedOld0, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung1, RemappedOld0, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung1, RemappedOld0, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung1, RemappedOld0, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung1, RemappedOld0, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung1, RemappedOld0, MarkedYoung1, MarkedOld1);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung1, RemappedOld0, MarkedYoung1, MarkedOld1);
+
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung1, RemappedOld1, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung1, RemappedOld1, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung1, RemappedOld1, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung1, RemappedOld1, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung1, RemappedOld1, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung1, RemappedOld1, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered0, RemappedYoung1, RemappedOld1, MarkedYoung1, MarkedOld1);
+    test_is_checks_on(null_value, Remembered0, RemappedYoung1, RemappedOld1, MarkedYoung1, MarkedOld1);
+
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung0, RemappedOld0, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung0, RemappedOld0, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung0, RemappedOld0, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung0, RemappedOld0, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung0, RemappedOld0, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung0, RemappedOld0, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung0, RemappedOld0, MarkedYoung1, MarkedOld1);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung0, RemappedOld0, MarkedYoung1, MarkedOld1);
+
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung0, RemappedOld1, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung0, RemappedOld1, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung0, RemappedOld1, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung0, RemappedOld1, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung0, RemappedOld1, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung0, RemappedOld1, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung0, RemappedOld1, MarkedYoung1, MarkedOld1);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung0, RemappedOld1, MarkedYoung1, MarkedOld1);
+
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung1, RemappedOld0, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung1, RemappedOld0, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung1, RemappedOld0, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung1, RemappedOld0, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung1, RemappedOld0, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung1, RemappedOld0, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung1, RemappedOld0, MarkedYoung1, MarkedOld1);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung1, RemappedOld0, MarkedYoung1, MarkedOld1);
+
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung1, RemappedOld1, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung1, RemappedOld1, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung1, RemappedOld1, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung1, RemappedOld1, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung1, RemappedOld1, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung1, RemappedOld1, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered1, RemappedYoung1, RemappedOld1, MarkedYoung1, MarkedOld1);
+    test_is_checks_on(null_value, Remembered1, RemappedYoung1, RemappedOld1, MarkedYoung1, MarkedOld1);
+
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung0, RemappedOld0, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung0, RemappedOld0, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung0, RemappedOld0, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung0, RemappedOld0, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung0, RemappedOld0, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung0, RemappedOld0, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung0, RemappedOld0, MarkedYoung1, MarkedOld1);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung0, RemappedOld0, MarkedYoung1, MarkedOld1);
+
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung0, RemappedOld1, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung0, RemappedOld1, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung0, RemappedOld1, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung0, RemappedOld1, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung0, RemappedOld1, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung0, RemappedOld1, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung0, RemappedOld1, MarkedYoung1, MarkedOld1);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung0, RemappedOld1, MarkedYoung1, MarkedOld1);
+
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung1, RemappedOld0, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung1, RemappedOld0, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung1, RemappedOld0, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung1, RemappedOld0, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung1, RemappedOld0, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung1, RemappedOld0, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung1, RemappedOld0, MarkedYoung1, MarkedOld1);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung1, RemappedOld0, MarkedYoung1, MarkedOld1);
+
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung1, RemappedOld1, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung1, RemappedOld1, MarkedYoung0, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung1, RemappedOld1, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung1, RemappedOld1, MarkedYoung0, MarkedOld1);
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung1, RemappedOld1, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung1, RemappedOld1, MarkedYoung1, MarkedOld0);
+    test_is_checks_on(valid_value, Remembered11, RemappedYoung1, RemappedOld1, MarkedYoung1, MarkedOld1);
+    test_is_checks_on(null_value, Remembered11, RemappedYoung1, RemappedOld1, MarkedYoung1, MarkedOld1);
+
+    test_is_checks_on(null_value, Uncolored, Uncolored, Uncolored, Uncolored, Uncolored);
+  }
+
+  static void advance_and_test_young_phase(int& phase, int amount) {
+    for (int i = 0; i < amount; ++i) {
+      if (++phase & 1) {
+        ZGlobalsPointers::flip_young_mark_start();
+      } else {
+        ZGlobalsPointers::flip_young_relocate_start();
+      }
+      test_is_checks_on_all();
+    }
+  }
+
+  static void advance_and_test_old_phase(int& phase, int amount) {
+    for (int i = 0; i < amount; ++i) {
+      if (++phase & 1) {
+        ZGlobalsPointers::flip_old_mark_start();
+      } else {
+        ZGlobalsPointers::flip_old_relocate_start();
+      }
+      test_is_checks_on_all();
+    }
+  }
+
+  static void is_checks() {
+    int young_phase = 0;
+    int old_phase = 0;
     // Setup
-    ZAddress::initialize();
-    ZAddress::flip_to_marked();
+    ZGlobalsPointers::initialize();
+    test_is_checks_on_all();
 
-    // Test that a normal good pointer is good and weak good, but not finalizable
-    const uintptr_t addr1 = ZAddress::good(1);
-    EXPECT_FALSE(ZAddress::is_finalizable(addr1));
-    EXPECT_TRUE(ZAddress::is_marked(addr1));
-    EXPECT_FALSE(ZAddress::is_remapped(addr1));
-    EXPECT_TRUE(ZAddress::is_weak_good(addr1));
-    EXPECT_TRUE(ZAddress::is_weak_good_or_null(addr1));
-    EXPECT_TRUE(ZAddress::is_good(addr1));
-    EXPECT_TRUE(ZAddress::is_good_or_null(addr1));
+    advance_and_test_old_phase(old_phase, 4);
+    advance_and_test_young_phase(young_phase, 4);
 
-    // Test that a finalizable good pointer is finalizable and weak good, but not good
-    const uintptr_t addr2 = ZAddress::finalizable_good(1);
-    EXPECT_TRUE(ZAddress::is_finalizable(addr2));
-    EXPECT_TRUE(ZAddress::is_marked(addr2));
-    EXPECT_FALSE(ZAddress::is_remapped(addr2));
-    EXPECT_TRUE(ZAddress::is_weak_good(addr2));
-    EXPECT_TRUE(ZAddress::is_weak_good_or_null(addr2));
-    EXPECT_FALSE(ZAddress::is_good(addr2));
-    EXPECT_FALSE(ZAddress::is_good_or_null(addr2));
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 4);
 
-    // Flip to remapped and test that it's no longer weak good
-    ZAddress::flip_to_remapped();
-    EXPECT_TRUE(ZAddress::is_finalizable(addr2));
-    EXPECT_TRUE(ZAddress::is_marked(addr2));
-    EXPECT_FALSE(ZAddress::is_remapped(addr2));
-    EXPECT_FALSE(ZAddress::is_weak_good(addr2));
-    EXPECT_FALSE(ZAddress::is_weak_good_or_null(addr2));
-    EXPECT_FALSE(ZAddress::is_good(addr2));
-    EXPECT_FALSE(ZAddress::is_good_or_null(addr2));
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 4);
+
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 4);
+
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 4);
+
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 3);
+
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 3);
+
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 3);
+
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 3);
+
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 2);
+
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 2);
+
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 2);
+
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 2);
+
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 1);
+
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 1);
+
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 1);
+
+    advance_and_test_old_phase(old_phase, 1);
+    advance_and_test_young_phase(young_phase, 1);
   }
 };
 
-TEST_F(ZAddressTest, is_good) {
-  is_good_bit(ZAddressMetadataMarked0);
-  is_good_bit(ZAddressMetadataMarked1);
-  is_good_bit(ZAddressMetadataRemapped);
-}
-
-TEST_F(ZAddressTest, is_good_or_null) {
-  is_good_or_null_bit(ZAddressMetadataMarked0);
-  is_good_or_null_bit(ZAddressMetadataMarked1);
-  is_good_or_null_bit(ZAddressMetadataRemapped);
-}
-
-TEST_F(ZAddressTest, is_weak_good_or_null) {
-#define check_is_weak_good_or_null(value)                                        \
-  EXPECT_EQ(ZAddress::is_weak_good_or_null(value),                               \
-            (ZAddress::is_good_or_null(value) || ZAddress::is_remapped(value)))  \
-    << "is_good_or_null: " << ZAddress::is_good_or_null(value)                   \
-    << " is_remaped: " << ZAddress::is_remapped(value)                           \
-    << " is_good_or_null_or_remapped: " << ZAddress::is_weak_good_or_null(value)
-
-  check_is_weak_good_or_null((uintptr_t)NULL);
-  check_is_weak_good_or_null(ZAddressMetadataMarked0);
-  check_is_weak_good_or_null(ZAddressMetadataMarked1);
-  check_is_weak_good_or_null(ZAddressMetadataRemapped);
-  check_is_weak_good_or_null((uintptr_t)0x123);
-}
-
-TEST_F(ZAddressTest, finalizable) {
-  finalizable();
+TEST_F(ZAddressTest, is_checks) {
+  is_checks();
 }
diff --git a/test/hotspot/gtest/gc/z/test_zForwarding.cpp b/test/hotspot/gtest/gc/z/test_zForwarding.cpp
index 1eb8b7c4747..622c6d9d8f4 100644
--- a/test/hotspot/gtest/gc/z/test_zForwarding.cpp
+++ b/test/hotspot/gtest/gc/z/test_zForwarding.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,8 +25,11 @@
 #include "gc/z/zAddress.inline.hpp"
 #include "gc/z/zForwarding.inline.hpp"
 #include "gc/z/zForwardingAllocator.inline.hpp"
+#include "gc/z/zGeneration.inline.hpp"
 #include "gc/z/zGlobals.hpp"
+#include "gc/z/zHeap.hpp"
 #include "gc/z/zPage.inline.hpp"
+#include "runtime/os.hpp"
 #include "unittest.hpp"
 
 using namespace testing;
@@ -39,6 +42,80 @@ using namespace testing;
 
 class ZForwardingTest : public Test {
 public:
+  // Setup and tear down
+  ZHeap*            _old_heap;
+  ZGenerationOld*   _old_old;
+  ZGenerationYoung* _old_young;
+  char*             _reserved;
+  static size_t     _page_offset;
+
+  char* reserve_page_memory() {
+    // Probe for a free 2MB region inside the usable address range.
+    // Inspired by ZVirtualMemoryManager::reserve_contiguous.
+    const size_t unused = ZAddressOffsetMax - ZGranuleSize;
+    const size_t increment = MAX2(align_up(unused / 100, ZGranuleSize), ZGranuleSize);
+
+    for (uintptr_t start = 0; start + ZGranuleSize <= ZAddressOffsetMax; start += increment) {
+      char* const reserved = os::attempt_reserve_memory_at((char*)ZAddressHeapBase + start, ZGranuleSize, false /* executable */);
+      if (reserved != nullptr) {
+        // Success
+        return reserved;
+      }
+    }
+
+    // Failed
+    return nullptr;
+  }
+
+  virtual void SetUp() {
+    ZGlobalsPointers::initialize();
+    _old_heap = ZHeap::_heap;
+    ZHeap::_heap = (ZHeap*)os::malloc(sizeof(ZHeap), mtTest);
+
+    _old_old = ZGeneration::_old;
+    _old_young = ZGeneration::_young;
+
+    ZGeneration::_old = &ZHeap::_heap->_old;
+    ZGeneration::_young = &ZHeap::_heap->_young;
+
+    *const_cast<ZGenerationId*>(&ZGeneration::_old->_id) = ZGenerationId::old;
+    *const_cast<ZGenerationId*>(&ZGeneration::_young->_id) = ZGenerationId::young;
+
+    ZGeneration::_old->_seqnum = 1;
+    ZGeneration::_young->_seqnum = 2;
+
+    // Preconditions for reserve_free_granule()
+    ASSERT_NE(ZAddressHeapBase, 0u);
+    ASSERT_NE(ZAddressOffsetMax, 0u);
+    ASSERT_NE(ZGranuleSize, 0u);
+
+    _reserved = nullptr;
+
+    // Find a suitable address for the testing page
+    char* reserved = reserve_page_memory();
+
+    ASSERT_NE(reserved, nullptr) << "Failed to reserve the page granule. Test needs tweaking";
+    ASSERT_GE(reserved, (char*)ZAddressHeapBase);
+    ASSERT_LT(reserved, (char*)ZAddressHeapBase + ZAddressOffsetMax);
+
+    _reserved = reserved;
+
+    os::commit_memory((char*)_reserved, ZGranuleSize, false /* executable */);
+
+    _page_offset = uintptr_t(_reserved) - ZAddressHeapBase;
+  }
+
+  virtual void TearDown() {
+    os::free(ZHeap::_heap);
+    ZHeap::_heap = _old_heap;
+    ZGeneration::_old = _old_old;
+    ZGeneration::_young = _old_young;
+    if (_reserved != nullptr) {
+      os::uncommit_memory((char*)_reserved, ZGranuleSize, false /* executable */);
+      os::release_memory((char*)_reserved, ZGranuleSize);
+    }
+  }
+
   // Helper functions
 
   class SequenceToFromIndex : AllStatic {
@@ -85,7 +162,7 @@ public:
       ZForwardingEntry entry = forwarding->find(from_index, &cursor);
       ASSERT_FALSE(entry.populated()) << CAPTURE2(from_index, size);
 
-      forwarding->insert(from_index, from_index, &cursor);
+      forwarding->insert(from_index, zoffset(from_index), &cursor);
     }
 
     // Verify
@@ -113,7 +190,7 @@ public:
       ZForwardingEntry entry = forwarding->find(from_index, &cursor);
       ASSERT_FALSE(entry.populated()) << CAPTURE2(from_index, size);
 
-      forwarding->insert(from_index, from_index, &cursor);
+      forwarding->insert(from_index, zoffset(from_index), &cursor);
     }
 
     // Verify populated even indices
@@ -144,19 +221,27 @@ public:
 
   static void test(void (*function)(ZForwarding*), uint32_t size) {
     // Create page
-    const ZVirtualMemory vmem(0, ZPageSizeSmall);
-    const ZPhysicalMemory pmem(ZPhysicalMemorySegment(0, ZPageSizeSmall, true));
-    ZPage page(ZPageTypeSmall, vmem, pmem);
+    const ZVirtualMemory vmem(zoffset(_page_offset), ZPageSizeSmall);
+    const ZPhysicalMemory pmem(ZPhysicalMemorySegment(zoffset(0), ZPageSizeSmall, true));
+    ZPage page(ZPageType::small, vmem, pmem);
 
-    page.reset();
+    page.reset(ZPageAge::eden, ZPageResetType::Allocation);
 
     const size_t object_size = 16;
-    const uintptr_t object = page.alloc_object(object_size);
+    const zaddress object = page.alloc_object(object_size);
 
-    ZGlobalSeqNum++;
+    ZGeneration::young()->_seqnum++;
 
-    bool dummy = false;
-    page.mark_object(ZAddress::marked(object), dummy, dummy);
+    ZGeneration::young()->set_phase(ZGeneration::Phase::Mark);
+    ZGeneration::young()->set_phase(ZGeneration::Phase::MarkComplete);
+    ZGeneration::young()->set_phase(ZGeneration::Phase::Relocate);
+
+    //page.mark_object(object, dummy, dummy);
+    {
+      bool dummy = false;
+      const BitMap::idx_t index = page.bit_index(object);
+      page._livemap.set(page._generation_id, index, dummy, dummy);
+    }
 
     const uint32_t live_objects = size;
     const size_t live_bytes = live_objects * object_size;
@@ -168,7 +253,7 @@ public:
     allocator.reset((sizeof(ZForwarding)) + (nentries * sizeof(ZForwardingEntry)));
 
     // Setup forwarding
-    ZForwarding* const forwarding = ZForwarding::alloc(&allocator, &page);
+    ZForwarding* const forwarding = ZForwarding::alloc(&allocator, &page, ZPageAge::survivor1);
 
     // Actual test function
     (*function)(forwarding);
@@ -188,18 +273,20 @@ public:
   }
 };
 
-TEST_F(ZForwardingTest, setup) {
+TEST_VM_F(ZForwardingTest, setup) {
   test(&ZForwardingTest::setup);
 }
 
-TEST_F(ZForwardingTest, find_empty) {
+TEST_VM_F(ZForwardingTest, find_empty) {
   test(&ZForwardingTest::find_empty);
 }
 
-TEST_F(ZForwardingTest, find_full) {
+TEST_VM_F(ZForwardingTest, find_full) {
   test(&ZForwardingTest::find_full);
 }
 
-TEST_F(ZForwardingTest, find_every_other) {
+TEST_VM_F(ZForwardingTest, find_every_other) {
   test(&ZForwardingTest::find_every_other);
 }
+
+size_t ZForwardingTest::_page_offset;
diff --git a/test/hotspot/gtest/gc/z/test_zIndexDistributor.cpp b/test/hotspot/gtest/gc/z/test_zIndexDistributor.cpp
new file mode 100644
index 00000000000..1a18b8fec9e
--- /dev/null
+++ b/test/hotspot/gtest/gc/z/test_zIndexDistributor.cpp
@@ -0,0 +1,171 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zIndexDistributor.inline.hpp"
+#include "unittest.hpp"
+
+class ZIndexDistributorTest : public ::testing::Test {
+protected:
+  static void test_claim_tree_claim_level_size() {
+    // max_index: 16, 16, 16, rest
+    // claim level: 1, 16, 16 * 16, 16 * 16 * 16
+    ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_size(0), 1);
+    ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_size(1), 16);
+    ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_size(2), 16 * 16);
+    ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_size(3), 16 * 16 * 16);
+  }
+
+  static void test_claim_tree_claim_level_end_index() {
+    // First level is padded
+    ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_end_index(0), 16);
+    ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_end_index(1), 16 + 16);
+    ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_end_index(2), 16 + 16 + 16 * 16);
+    ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_end_index(3), 16 + 16 + 16 * 16 + 16 * 16 * 16);
+  }
+
+  static void test_claim_tree_claim_index() {
+    // First level should always give index 0
+    {
+      int indices[4] = {0, 0, 0, 0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_index(indices, 0), 0);
+    }
+    {
+      int indices[4] = {1, 0, 0, 0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_index(indices, 0), 0);
+    }
+    {
+      int indices[4] = {15, 0, 0, 0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_index(indices, 0), 0);
+    }
+    {
+      int indices[4] = {16, 0, 0, 0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_index(indices, 0), 0);
+    }
+
+    // Second level should depend on first claimed index
+
+    // Second-level start after first-level padding
+    const int second_level_start = 16;
+
+    {
+      int indices[4] = {0, 0, 0, 0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_index(indices, 1), second_level_start);
+    }
+    {
+      int indices[4] = {1, 0, 0, 0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_index(indices, 1), second_level_start + 1);
+    }
+    {
+      int indices[4] = {15, 0, 0, 0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_index(indices, 1), second_level_start + 15);
+    }
+
+    // Third level
+
+    const int third_level_start = second_level_start + 16;
+
+    {
+      int indices[4] = {0, 0, 0, 0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_index(indices, 2), third_level_start);
+    }
+    {
+      int indices[4] = {1, 0, 0, 0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_index(indices, 2), third_level_start + 1 * 16);
+    }
+    {
+      int indices[4] = {15, 0, 0, 0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_index(indices, 2), third_level_start + 15 * 16);
+    }
+    {
+      int indices[4] = {1, 2, 0, 0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_index(indices, 2), third_level_start + 1 * 16 + 2);
+    }
+    {
+      int indices[4] = {15, 14, 0, 0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_index(indices, 2), third_level_start + 15 * 16 + 14);
+    }
+
+  }
+
+  static void test_claim_tree_claim_level_index() {
+    {
+      int indices[4] = {0,0,0,0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_index(indices, 1), 0);
+    }
+    {
+      int indices[4] = {1,0,0,0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_index(indices, 1), 1);
+    }
+
+    {
+      int indices[4] = {0,0,0,0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_index(indices, 2), 0);
+    }
+    {
+      int indices[4] = {1,0,0,0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_index(indices, 2), 1 * 16);
+    }
+    {
+      int indices[4] = {2,0,0,0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_index(indices, 2), 2 * 16);
+    }
+    {
+      int indices[4] = {2,1,0,0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_index(indices, 2), 2 * 16 + 1);
+    }
+
+    {
+      int indices[4] = {0,0,0,0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_index(indices, 3), 0);
+    }
+    {
+      int indices[4] = {1,0,0,0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_index(indices, 3), 1 * 16 * 16);
+    }
+    {
+      int indices[4] = {1,2,0,0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_index(indices, 3), 1 * 16 * 16 + 2 * 16);
+    }
+    {
+      int indices[4] = {1,2,1,0};
+      ASSERT_EQ(ZIndexDistributorClaimTree::claim_level_index(indices, 3), 1 * 16 * 16 + 2 * 16 + 1);
+    }
+  }
+};
+
+TEST_F(ZIndexDistributorTest, test_claim_tree_claim_level_size) {
+  test_claim_tree_claim_level_size();
+}
+
+TEST_F(ZIndexDistributorTest, test_claim_tree_claim_level_end_index) {
+  test_claim_tree_claim_level_end_index();
+}
+
+TEST_F(ZIndexDistributorTest, test_claim_tree_claim_level_index) {
+  test_claim_tree_claim_level_index();
+}
+
+TEST_F(ZIndexDistributorTest, test_claim_tree_claim_index) {
+  test_claim_tree_claim_index();
+}
diff --git a/test/hotspot/gtest/gc/z/test_zLiveMap.cpp b/test/hotspot/gtest/gc/z/test_zLiveMap.cpp
index 53b402d320d..821273b9b60 100644
--- a/test/hotspot/gtest/gc/z/test_zLiveMap.cpp
+++ b/test/hotspot/gtest/gc/z/test_zLiveMap.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -22,29 +22,64 @@
  */
 
 #include "precompiled.hpp"
+#include "gc/z/zGenerationId.hpp"
+#include "gc/z/zGlobals.hpp"
 #include "gc/z/zLiveMap.inline.hpp"
 #include "unittest.hpp"
 
 class ZLiveMapTest : public ::testing::Test {
+private:
+  // Setup and tear down
+  ZHeap*            _old_heap;
+  ZGenerationOld*   _old_old;
+  ZGenerationYoung* _old_young;
+
+public:
+
+  virtual void SetUp() {
+    ZGlobalsPointers::initialize();
+    _old_heap = ZHeap::_heap;
+    ZHeap::_heap = (ZHeap*)os::malloc(sizeof(ZHeap), mtTest);
+
+    _old_old = ZGeneration::_old;
+    _old_young = ZGeneration::_young;
+
+    ZGeneration::_old = &ZHeap::_heap->_old;
+    ZGeneration::_young = &ZHeap::_heap->_young;
+
+    *const_cast<ZGenerationId*>(&ZGeneration::_old->_id) = ZGenerationId::old;
+    *const_cast<ZGenerationId*>(&ZGeneration::_young->_id) = ZGenerationId::young;
+
+    ZGeneration::_old->_seqnum = 1;
+    ZGeneration::_young->_seqnum = 2;
+  }
+
+  virtual void TearDown() {
+    os::free(ZHeap::_heap);
+    ZHeap::_heap = _old_heap;
+    ZGeneration::_old = _old_old;
+    ZGeneration::_young = _old_young;
+  }
+
 protected:
   static void strongly_live_for_large_zpage() {
     // Large ZPages only have room for one object.
     ZLiveMap livemap(1);
 
     bool inc_live;
-    uintptr_t object = 0u;
+    BitMap::idx_t object_index = BitMap::idx_t(0);
 
     // Mark the object strong.
-    livemap.set(object, false /* finalizable */, inc_live);
+    livemap.set(ZGenerationId::old, object_index, false /* finalizable */, inc_live);
 
     // Check that both bits are in the same segment.
     ASSERT_EQ(livemap.index_to_segment(0), livemap.index_to_segment(1));
 
     // Check that the object was marked.
-    ASSERT_TRUE(livemap.get(0));
+    ASSERT_TRUE(livemap.get(ZGenerationId::old, 0));
 
     // Check that the object was strongly marked.
-    ASSERT_TRUE(livemap.get(1));
+    ASSERT_TRUE(livemap.get(ZGenerationId::old, 1));
 
     ASSERT_TRUE(inc_live);
   }
diff --git a/test/hotspot/gtest/gc/z/test_zMemory.cpp b/test/hotspot/gtest/gc/z/test_zMemory.cpp
new file mode 100644
index 00000000000..6aa7f460b4d
--- /dev/null
+++ b/test/hotspot/gtest/gc/z/test_zMemory.cpp
@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include "precompiled.hpp"
+#include "gc/z/zGlobals.hpp"
+#include "gc/z/zMemory.inline.hpp"
+#include "unittest.hpp"
+
+class ZAddressOffsetMaxSetter {
+private:
+  const size_t _old_max;
+  const size_t _old_mask;
+
+public:
+  ZAddressOffsetMaxSetter() :
+      _old_max(ZAddressOffsetMax),
+      _old_mask(ZAddressOffsetMask) {
+    ZAddressOffsetMax = size_t(16) * G * 1024;
+    ZAddressOffsetMask = ZAddressOffsetMax - 1;
+  }
+  ~ZAddressOffsetMaxSetter() {
+    ZAddressOffsetMax = _old_max;
+    ZAddressOffsetMask = _old_mask;
+  }
+};
+
+TEST(ZMemory, accessors) {
+  ZAddressOffsetMaxSetter setter;
+
+  {
+    ZMemory mem(zoffset(0), ZGranuleSize);
+
+    EXPECT_EQ(mem.start(), zoffset(0));
+    EXPECT_EQ(mem.end(), zoffset_end(ZGranuleSize));
+    EXPECT_EQ(mem.size(), ZGranuleSize);
+  }
+
+
+  {
+    ZMemory mem(zoffset(ZGranuleSize), ZGranuleSize);
+
+    EXPECT_EQ(mem.start(), zoffset(ZGranuleSize));
+    EXPECT_EQ(mem.end(), zoffset_end(ZGranuleSize + ZGranuleSize));
+    EXPECT_EQ(mem.size(), ZGranuleSize);
+  }
+
+  {
+    // Max area - check end boundary
+    ZMemory mem(zoffset(0), ZAddressOffsetMax);
+
+    EXPECT_EQ(mem.start(), zoffset(0));
+    EXPECT_EQ(mem.end(), zoffset_end(ZAddressOffsetMax));
+    EXPECT_EQ(mem.size(), ZAddressOffsetMax);
+  }
+}
+
+TEST(ZMemory, resize) {
+  ZAddressOffsetMaxSetter setter;
+
+  ZMemory mem(zoffset(ZGranuleSize * 2), ZGranuleSize * 2) ;
+
+  mem.shrink_from_front(ZGranuleSize);
+  EXPECT_EQ(mem.start(),   zoffset(ZGranuleSize * 3));
+  EXPECT_EQ(mem.end(), zoffset_end(ZGranuleSize * 4));
+  EXPECT_EQ(mem.size(),            ZGranuleSize * 1);
+  mem.grow_from_front(ZGranuleSize);
+
+  mem.shrink_from_back(ZGranuleSize);
+  EXPECT_EQ(mem.start(),   zoffset(ZGranuleSize * 2));
+  EXPECT_EQ(mem.end(), zoffset_end(ZGranuleSize * 3));
+  EXPECT_EQ(mem.size(),            ZGranuleSize * 1);
+  mem.grow_from_back(ZGranuleSize);
+
+  mem.grow_from_front(ZGranuleSize);
+  EXPECT_EQ(mem.start(),   zoffset(ZGranuleSize * 1));
+  EXPECT_EQ(mem.end(), zoffset_end(ZGranuleSize * 4));
+  EXPECT_EQ(mem.size(),            ZGranuleSize * 3);
+  mem.shrink_from_front(ZGranuleSize);
+
+  mem.grow_from_back(ZGranuleSize);
+  EXPECT_EQ(mem.start(),   zoffset(ZGranuleSize * 2));
+  EXPECT_EQ(mem.end(), zoffset_end(ZGranuleSize * 5));
+  EXPECT_EQ(mem.size(),            ZGranuleSize * 3);
+  mem.shrink_from_back(ZGranuleSize);
+}
diff --git a/test/hotspot/gtest/gc/z/test_zPhysicalMemory.cpp b/test/hotspot/gtest/gc/z/test_zPhysicalMemory.cpp
index 51b88730b10..69861416928 100644
--- a/test/hotspot/gtest/gc/z/test_zPhysicalMemory.cpp
+++ b/test/hotspot/gtest/gc/z/test_zPhysicalMemory.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,8 +26,8 @@
 #include "unittest.hpp"
 
 TEST(ZPhysicalMemoryTest, copy) {
-  const ZPhysicalMemorySegment seg0(0, 100, true);
-  const ZPhysicalMemorySegment seg1(200, 100, true);
+  const ZPhysicalMemorySegment seg0(zoffset(0), 100, true);
+  const ZPhysicalMemorySegment seg1(zoffset(200), 100, true);
 
   ZPhysicalMemory pmem0;
   pmem0.add_segment(seg0);
@@ -52,13 +52,13 @@ TEST(ZPhysicalMemoryTest, copy) {
 }
 
 TEST(ZPhysicalMemoryTest, add) {
-  const ZPhysicalMemorySegment seg0(0, 1, true);
-  const ZPhysicalMemorySegment seg1(1, 1, true);
-  const ZPhysicalMemorySegment seg2(2, 1, true);
-  const ZPhysicalMemorySegment seg3(3, 1, true);
-  const ZPhysicalMemorySegment seg4(4, 1, true);
-  const ZPhysicalMemorySegment seg5(5, 1, true);
-  const ZPhysicalMemorySegment seg6(6, 1, true);
+  const ZPhysicalMemorySegment seg0(zoffset(0), 1, true);
+  const ZPhysicalMemorySegment seg1(zoffset(1), 1, true);
+  const ZPhysicalMemorySegment seg2(zoffset(2), 1, true);
+  const ZPhysicalMemorySegment seg3(zoffset(3), 1, true);
+  const ZPhysicalMemorySegment seg4(zoffset(4), 1, true);
+  const ZPhysicalMemorySegment seg5(zoffset(5), 1, true);
+  const ZPhysicalMemorySegment seg6(zoffset(6), 1, true);
 
   ZPhysicalMemory pmem0;
   EXPECT_EQ(pmem0.nsegments(), 0);
@@ -116,9 +116,9 @@ TEST(ZPhysicalMemoryTest, add) {
 TEST(ZPhysicalMemoryTest, remove) {
   ZPhysicalMemory pmem;
 
-  pmem.add_segment(ZPhysicalMemorySegment(10, 10, true));
-  pmem.add_segment(ZPhysicalMemorySegment(30, 10, true));
-  pmem.add_segment(ZPhysicalMemorySegment(50, 10, true));
+  pmem.add_segment(ZPhysicalMemorySegment(zoffset(10), 10, true));
+  pmem.add_segment(ZPhysicalMemorySegment(zoffset(30), 10, true));
+  pmem.add_segment(ZPhysicalMemorySegment(zoffset(50), 10, true));
   EXPECT_EQ(pmem.nsegments(), 3);
   EXPECT_EQ(pmem.size(), 30u);
   EXPECT_FALSE(pmem.is_null());
@@ -132,9 +132,9 @@ TEST(ZPhysicalMemoryTest, remove) {
 TEST(ZPhysicalMemoryTest, split) {
   ZPhysicalMemory pmem;
 
-  pmem.add_segment(ZPhysicalMemorySegment(0, 10, true));
-  pmem.add_segment(ZPhysicalMemorySegment(10, 10, true));
-  pmem.add_segment(ZPhysicalMemorySegment(30, 10, true));
+  pmem.add_segment(ZPhysicalMemorySegment(zoffset(0), 10, true));
+  pmem.add_segment(ZPhysicalMemorySegment(zoffset(10), 10, true));
+  pmem.add_segment(ZPhysicalMemorySegment(zoffset(30), 10, true));
   EXPECT_EQ(pmem.nsegments(), 2);
   EXPECT_EQ(pmem.size(), 30u);
 
@@ -159,10 +159,10 @@ TEST(ZPhysicalMemoryTest, split) {
 
 TEST(ZPhysicalMemoryTest, split_committed) {
   ZPhysicalMemory pmem0;
-  pmem0.add_segment(ZPhysicalMemorySegment(0, 10, true));
-  pmem0.add_segment(ZPhysicalMemorySegment(10, 10, false));
-  pmem0.add_segment(ZPhysicalMemorySegment(20, 10, true));
-  pmem0.add_segment(ZPhysicalMemorySegment(30, 10, false));
+  pmem0.add_segment(ZPhysicalMemorySegment(zoffset(0), 10, true));
+  pmem0.add_segment(ZPhysicalMemorySegment(zoffset(10), 10, false));
+  pmem0.add_segment(ZPhysicalMemorySegment(zoffset(20), 10, true));
+  pmem0.add_segment(ZPhysicalMemorySegment(zoffset(30), 10, false));
   EXPECT_EQ(pmem0.nsegments(), 4);
   EXPECT_EQ(pmem0.size(), 40u);
 
diff --git a/test/hotspot/gtest/gc/z/test_zVirtualMemory.cpp b/test/hotspot/gtest/gc/z/test_zVirtualMemory.cpp
index 1c7ced8e948..70981fba38f 100644
--- a/test/hotspot/gtest/gc/z/test_zVirtualMemory.cpp
+++ b/test/hotspot/gtest/gc/z/test_zVirtualMemory.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2019, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,7 +26,7 @@
 #include "unittest.hpp"
 
 TEST(ZVirtualMemory, split) {
-  ZVirtualMemory vmem(0, 10);
+  ZVirtualMemory vmem(zoffset(0), 10);
 
   ZVirtualMemory vmem0 = vmem.split(0);
   EXPECT_EQ(vmem0.size(), 0u);
diff --git a/test/hotspot/gtest/runtime/test_vmStructs.cpp b/test/hotspot/gtest/runtime/test_vmStructs.cpp
index 3c6f84eceea..de9182e5f4d 100644
--- a/test/hotspot/gtest/runtime/test_vmStructs.cpp
+++ b/test/hotspot/gtest/runtime/test_vmStructs.cpp
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -58,7 +58,7 @@ TEST(VMStructs, VMTypes_duplicates)  {
   for (int i = 0; VMStructs::localHotSpotVMTypes[i].typeName != NULL; i++) {
     for (int j = i + 1; VMStructs::localHotSpotVMTypes[j].typeName != NULL; j++) {
       EXPECT_STRNE(VMStructs::localHotSpotVMTypes[i].typeName, VMStructs::localHotSpotVMTypes[j].typeName)
-              << "Duplicate entries on indexes " << i << " and " << j;
+              << "Duplicate entries on indexes " << i << " and " << j << " : " << VMStructs::localHotSpotVMTypes[i].typeName;
     }
   }
 }
diff --git a/test/hotspot/jtreg/ProblemList-generational-zgc.txt b/test/hotspot/jtreg/ProblemList-generational-zgc.txt
new file mode 100644
index 00000000000..de22024f8d7
--- /dev/null
+++ b/test/hotspot/jtreg/ProblemList-generational-zgc.txt
@@ -0,0 +1,117 @@
+#
+# Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+#############################################################################
+#
+# List of quarantined tests for testing with Generational ZGC.
+#
+#############################################################################
+
+# Quiet all SA tests
+
+resourcehogs/serviceability/sa/TestHeapDumpForLargeArray.java 8307393   generic-all
+serviceability/sa/CDSJMapClstats.java                         8307393   generic-all
+serviceability/sa/ClhsdbAttach.java                           8307393   generic-all
+serviceability/sa/ClhsdbCDSCore.java                          8307393   generic-all
+serviceability/sa/ClhsdbCDSJstackPrintAll.java                8307393   generic-all
+serviceability/sa/ClhsdbClasses.java                          8307393   generic-all
+serviceability/sa/ClhsdbDumpclass.java                        8307393   generic-all
+serviceability/sa/ClhsdbDumpheap.java                         8307393   generic-all
+serviceability/sa/ClhsdbField.java                            8307393   generic-all
+serviceability/sa/ClhsdbFindPC.java#apa                       8307393   generic-all
+serviceability/sa/ClhsdbFindPC.java#no-xcomp-core             8307393   generic-all
+serviceability/sa/ClhsdbFindPC.java#no-xcomp-process          8307393   generic-all
+serviceability/sa/ClhsdbFindPC.java#xcomp-core                8307393   generic-all
+serviceability/sa/ClhsdbFindPC.java#xcomp-process             8307393   generic-all
+serviceability/sa/ClhsdbFlags.java                            8307393   generic-all
+serviceability/sa/ClhsdbHistory.java                          8307393   generic-all
+serviceability/sa/ClhsdbInspect.java                          8307393   generic-all
+serviceability/sa/ClhsdbJdis.java                             8307393   generic-all
+serviceability/sa/ClhsdbJhisto.java                           8307393   generic-all
+serviceability/sa/ClhsdbJstack.java#id0                       8307393   generic-all
+serviceability/sa/ClhsdbJstack.java#id1                       8307393   generic-all
+serviceability/sa/ClhsdbJstackXcompStress.java                8307393   generic-all
+serviceability/sa/ClhsdbLauncher.java                         8307393   generic-all
+serviceability/sa/ClhsdbLongConstant.java                     8307393   generic-all
+serviceability/sa/ClhsdbPmap.java                             8307393   generic-all
+serviceability/sa/ClhsdbPmap.java#core                        8307393   generic-all
+serviceability/sa/ClhsdbPmap.java#process                     8307393   generic-all
+serviceability/sa/ClhsdbPrintAll.java                         8307393   generic-all
+serviceability/sa/ClhsdbPrintAs.java                          8307393   generic-all
+serviceability/sa/ClhsdbPrintStatics.java                     8307393   generic-all
+serviceability/sa/ClhsdbPstack.java#core                      8307393   generic-all
+serviceability/sa/ClhsdbPstack.java#process                   8307393   generic-all
+serviceability/sa/ClhsdbScanOops.java                         8307393   generic-all
+serviceability/sa/ClhsdbSource.java                           8307393   generic-all
+serviceability/sa/ClhsdbSymbol.java                           8307393   generic-all
+serviceability/sa/ClhsdbThread.java                           8307393   generic-all
+serviceability/sa/ClhsdbThreadContext.java                    8307393   generic-all
+serviceability/sa/ClhsdbVmStructsDump.java                    8307393   generic-all
+serviceability/sa/ClhsdbWhere.java                            8307393   generic-all
+serviceability/sa/DeadlockDetectionTest.java                  8307393   generic-all
+serviceability/sa/JhsdbThreadInfoTest.java                    8307393   generic-all
+serviceability/sa/LingeredAppSysProps.java                    8307393   generic-all
+serviceability/sa/LingeredAppWithDefaultMethods.java          8307393   generic-all
+serviceability/sa/LingeredAppWithEnum.java                    8307393   generic-all
+serviceability/sa/LingeredAppWithInterface.java               8307393   generic-all
+serviceability/sa/LingeredAppWithInvokeDynamic.java           8307393   generic-all
+serviceability/sa/LingeredAppWithLock.java                    8307393   generic-all
+serviceability/sa/LingeredAppWithNativeMethod.java            8307393   generic-all
+serviceability/sa/LingeredAppWithRecComputation.java          8307393   generic-all
+serviceability/sa/TestClassDump.java                          8307393   generic-all
+serviceability/sa/TestClhsdbJstackLock.java                   8307393   generic-all
+serviceability/sa/TestCpoolForInvokeDynamic.java              8307393   generic-all
+serviceability/sa/TestDefaultMethods.java                     8307393   generic-all
+serviceability/sa/TestG1HeapRegion.java                       8307393   generic-all
+serviceability/sa/TestHeapDumpForInvokeDynamic.java           8307393   generic-all
+serviceability/sa/TestInstanceKlassSize.java                  8307393   generic-all
+serviceability/sa/TestInstanceKlassSizeForInterface.java      8307393   generic-all
+serviceability/sa/TestIntConstant.java                        8307393   generic-all
+serviceability/sa/TestJhsdbJstackLineNumbers.java             8307393   generic-all
+serviceability/sa/TestJhsdbJstackLock.java                    8307393   generic-all
+serviceability/sa/TestJhsdbJstackMixed.java                   8307393   generic-all
+serviceability/sa/TestJmapCore.java                           8307393   generic-all
+serviceability/sa/TestJmapCoreMetaspace.java                  8307393   generic-all
+serviceability/sa/TestObjectAlignment.java                    8307393   generic-all
+serviceability/sa/TestObjectMonitorIterate.java               8307393   generic-all
+serviceability/sa/TestPrintMdo.java                           8307393   generic-all
+serviceability/sa/TestRevPtrsForInvokeDynamic.java            8307393   generic-all
+serviceability/sa/TestSysProps.java                           8307393   generic-all
+serviceability/sa/TestType.java                               8307393   generic-all
+serviceability/sa/TestUniverse.java                           8307393   generic-all
+serviceability/sa/UniqueVtableTest.java                       8307393   generic-all
+serviceability/sa/jmap-hprof/JMapHProfLargeHeapProc.java      8307393   generic-all
+serviceability/sa/jmap-hprof/JMapHProfLargeHeapTest.java      8307393   generic-all
+serviceability/sa/sadebugd/ClhsdbAttachToDebugServer.java     8307393   generic-all
+serviceability/sa/sadebugd/ClhsdbTestConnectArgument.java     8307393   generic-all
+serviceability/sa/sadebugd/DebugdConnectTest.java             8307393   generic-all
+serviceability/sa/sadebugd/DebugdUtils.java                   8307393   generic-all
+serviceability/sa/sadebugd/DisableRegistryTest.java           8307393   generic-all
+serviceability/sa/sadebugd/PmapOnDebugdTest.java              8307393   generic-all
+serviceability/sa/sadebugd/RunCommandOnServerTest.java        8307393   generic-all
+serviceability/sa/sadebugd/SADebugDTest.java                  8307393   generic-all
+
+vmTestbase/gc/gctests/MemoryEaterMT/MemoryEaterMT.java        8289582   windows-x64
+
+vmTestbase/nsk/monitoring/MemoryPoolMBean/isCollectionUsageThresholdExceeded/isexceeded002/TestDescription.java 8298302 generic-all
+vmTestbase/nsk/sysdict/vm/stress/chain/chain007/chain007.java 8298991 linux-x64
diff --git a/test/hotspot/jtreg/compiler/gcbarriers/TestZGCBarrierElision.java b/test/hotspot/jtreg/compiler/gcbarriers/TestZGCBarrierElision.java
new file mode 100644
index 00000000000..409f9593010
--- /dev/null
+++ b/test/hotspot/jtreg/compiler/gcbarriers/TestZGCBarrierElision.java
@@ -0,0 +1,440 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package compiler.gcbarriers;
+
+import compiler.lib.ir_framework.*;
+import java.lang.invoke.VarHandle;
+import java.lang.invoke.MethodHandles;
+import java.util.concurrent.ThreadLocalRandom;
+
+/**
+ * @test
+ * @summary Test that the ZGC barrier elision optimization does not elide
+ *          necessary barriers. The tests use volatile memory accesses and
+ *          blackholes to prevent C2 from simply optimizing them away.
+ * @library /test/lib /
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational
+ * @run driver compiler.gcbarriers.TestZGCBarrierElision test-correctness
+ */
+
+/**
+ * @test
+ * @summary Test that the ZGC barrier elision optimization elides unnecessary
+ *          barriers following simple allocation and domination rules.
+ * @library /test/lib /
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational & (vm.simpleArch == "x64" | vm.simpleArch == "aarch64")
+ * @run driver compiler.gcbarriers.TestZGCBarrierElision test-effectiveness
+ */
+
+class Inner {}
+
+class Outer {
+    volatile Inner field1;
+    volatile Inner field2;
+    Outer() {}
+}
+
+class Common {
+
+    static Inner inner = new Inner();
+    static Outer outer = new Outer();
+    static Outer outer2 = new Outer();
+    static Outer[] outerArray = new Outer[42];
+
+    static final VarHandle field1VarHandle;
+    static final VarHandle field2VarHandle;
+    static {
+        MethodHandles.Lookup l = MethodHandles.lookup();
+        try {
+            field1VarHandle = l.findVarHandle(Outer.class, "field1", Inner.class);
+            field2VarHandle = l.findVarHandle(Outer.class, "field2", Inner.class);
+        } catch (Exception e) {
+            throw new Error(e);
+        }
+    }
+    static final VarHandle outerArrayVarHandle =
+        MethodHandles.arrayElementVarHandle(Outer[].class);
+
+    static final String REMAINING = "strong";
+    static final String ELIDED = "elided";
+
+    static void blackhole(Object o) {}
+    static void nonInlinedMethod() {}
+}
+
+public class TestZGCBarrierElision {
+
+    public static void main(String[] args) {
+        if (args.length != 1) {
+            throw new IllegalArgumentException();
+        }
+        Class testClass;
+        if (args[0].equals("test-correctness")) {
+            testClass = TestZGCCorrectBarrierElision.class;
+        } else if (args[0].equals("test-effectiveness")) {
+            testClass = TestZGCEffectiveBarrierElision.class;
+        } else {
+            throw new IllegalArgumentException();
+        }
+        String commonName = Common.class.getName();
+        TestFramework test = new TestFramework(testClass);
+        test.addFlags("-XX:+UseZGC", "-XX:+UnlockExperimentalVMOptions",
+                      "-XX:CompileCommand=blackhole," + commonName + "::blackhole",
+                      "-XX:CompileCommand=dontinline," + commonName + "::nonInlinedMethod",
+                      "-XX:LoopMaxUnroll=0");
+        test.start();
+    }
+}
+
+class TestZGCCorrectBarrierElision {
+
+    @Test
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_LOAD_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" },  phase = CompilePhase.FINAL_CODE)
+    static void testLoadThenStore(Outer o, Inner i) {
+        Common.blackhole(o.field1);
+        o.field1 = i;
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_LOAD_P_WITH_BARRIER_FLAG, Common.REMAINING, "2" }, phase = CompilePhase.FINAL_CODE)
+    static void testLoadThenLoadAnotherField(Outer o) {
+        Common.blackhole(o.field1);
+        Common.blackhole(o.field2);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_LOAD_P_WITH_BARRIER_FLAG, Common.REMAINING, "2" }, phase = CompilePhase.FINAL_CODE)
+    static void testLoadThenLoadFromAnotherObject(Outer o1, Outer o2) {
+        Common.blackhole(o1.field1);
+        Common.blackhole(o2.field1);
+    }
+
+    @Run(test = {"testLoadThenStore",
+                 "testLoadThenLoadAnotherField",
+                 "testLoadThenLoadFromAnotherObject"})
+    void runBasicTests() {
+        testLoadThenStore(Common.outer, Common.inner);
+        testLoadThenLoadAnotherField(Common.outer);
+        testLoadThenLoadFromAnotherObject(Common.outer, Common.outer2);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_LOAD_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testArrayLoadThenStore(Outer[] a, Outer o) {
+        Common.blackhole(Common.outerArrayVarHandle.getVolatile(a, 0));
+        Common.outerArrayVarHandle.setVolatile(a, 0, o);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_LOAD_P_WITH_BARRIER_FLAG, Common.REMAINING, "2" }, phase = CompilePhase.FINAL_CODE)
+    static void testArrayLoadThenLoadAnotherElement(Outer[] a) {
+        Common.blackhole(Common.outerArrayVarHandle.getVolatile(a, 0));
+        Common.blackhole(Common.outerArrayVarHandle.getVolatile(a, 10));
+    }
+
+    @Run(test = {"testArrayLoadThenStore",
+                 "testArrayLoadThenLoadAnotherElement"})
+    void runArrayTests() {
+        testArrayLoadThenStore(Common.outerArray, Common.outer);
+        testArrayLoadThenLoadAnotherElement(Common.outerArray);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.REMAINING, "2" }, phase = CompilePhase.FINAL_CODE)
+    static void testConditionalStoreThenStore(Outer o, Inner i, int value) {
+        if (value % 2 == 0) {
+            o.field1 = i;
+        }
+        o.field1 = i;
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.REMAINING, "2" }, phase = CompilePhase.FINAL_CODE)
+    static void testStoreThenCallThenStore(Outer o, Inner i) {
+        o.field1 = i;
+        Common.nonInlinedMethod();
+        o.field1 = i;
+    }
+
+    @Run(test = {"testConditionalStoreThenStore",
+                 "testStoreThenCallThenStore"})
+    void runControlFlowTests() {
+        testConditionalStoreThenStore(Common.outer, Common.inner, ThreadLocalRandom.current().nextInt(0, 100));
+        testStoreThenCallThenStore(Common.outer, Common.inner);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_GET_AND_SET_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testAllocateThenAtomic(Inner i) {
+        Outer o = new Outer();
+        Common.blackhole(o);
+        Common.field1VarHandle.getAndSet​(o, i);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_LOAD_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_GET_AND_SET_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testLoadThenAtomic(Outer o, Inner i) {
+        Common.blackhole(o.field1);
+        Common.field1VarHandle.getAndSet​(o, i);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_GET_AND_SET_P_WITH_BARRIER_FLAG, Common.REMAINING, "2" }, phase = CompilePhase.FINAL_CODE)
+    static void testAtomicThenAtomicAnotherField(Outer o, Inner i) {
+        Common.field1VarHandle.getAndSet​(o, i);
+        Common.field2VarHandle.getAndSet​(o, i);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_GET_AND_SET_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testAllocateArrayThenAtomicAtKnownIndex(Outer o) {
+        Outer[] a = new Outer[42];
+        Common.blackhole(a);
+        Common.outerArrayVarHandle.getAndSet(a, 2, o);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_GET_AND_SET_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testAllocateArrayThenAtomicAtUnknownIndex(Outer o, int index) {
+        Outer[] a = new Outer[42];
+        Common.blackhole(a);
+        Common.outerArrayVarHandle.getAndSet(a, index, o);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_GET_AND_SET_P_WITH_BARRIER_FLAG, Common.REMAINING, "2" }, phase = CompilePhase.FINAL_CODE)
+    static void testArrayAtomicThenAtomicAtUnknownIndices(Outer[] a, Outer o, int index1, int index2) {
+        Common.outerArrayVarHandle.getAndSet(a, index1, o);
+        Common.outerArrayVarHandle.getAndSet(a, index2, o);
+    }
+
+    @Run(test = {"testAllocateThenAtomic",
+                 "testLoadThenAtomic",
+                 "testAtomicThenAtomicAnotherField",
+                 "testAllocateArrayThenAtomicAtKnownIndex",
+                 "testAllocateArrayThenAtomicAtUnknownIndex",
+                 "testArrayAtomicThenAtomicAtUnknownIndices"})
+    void runAtomicOperationTests() {
+        testAllocateThenAtomic(Common.inner);
+        testLoadThenAtomic(Common.outer, Common.inner);
+        testAtomicThenAtomicAnotherField(Common.outer, Common.inner);
+        testAllocateArrayThenAtomicAtKnownIndex(Common.outer);
+        testAllocateArrayThenAtomicAtUnknownIndex(Common.outer, 10);
+        testArrayAtomicThenAtomicAtUnknownIndices(Common.outerArray, Common.outer, 10, 20);
+    }
+}
+
+class TestZGCEffectiveBarrierElision {
+
+    @Test
+    @IR(counts = { IRNode.Z_LOAD_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testAllocateThenLoad() {
+        Outer o1 = new Outer();
+        Common.blackhole(o1);
+        // This load is directly optimized away by C2.
+        Common.blackhole(o1.field1);
+        Common.blackhole(o1.field1);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testAllocateThenStore(Inner i) {
+        Outer o1 = new Outer();
+        Common.blackhole(o1);
+        o1.field1 = i;
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_LOAD_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_LOAD_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testLoadThenLoad(Outer o) {
+        Common.blackhole(o.field1);
+        Common.blackhole(o.field1);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testStoreThenStore(Outer o, Inner i) {
+        o.field1 = i;
+        o.field1 = i;
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_LOAD_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" },  phase = CompilePhase.FINAL_CODE)
+    static void testStoreThenLoad(Outer o, Inner i) {
+        o.field1 = i;
+        Common.blackhole(o.field1);
+    }
+
+    @Run(test = {"testAllocateThenLoad",
+                 "testAllocateThenStore",
+                 "testLoadThenLoad",
+                 "testStoreThenStore",
+                 "testStoreThenLoad"})
+    void runBasicTests() {
+        testAllocateThenLoad();
+        testAllocateThenStore(Common.inner);
+        testLoadThenLoad(Common.outer);
+        testStoreThenStore(Common.outer, Common.inner);
+        testStoreThenLoad(Common.outer, Common.inner);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testAllocateArrayThenStoreAtKnownIndex(Outer o) {
+        Outer[] a = new Outer[42];
+        Common.blackhole(a);
+        Common.outerArrayVarHandle.setVolatile(a, 0, o);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testAllocateArrayThenStoreAtUnknownIndex(Outer o, int index) {
+        Outer[] a = new Outer[42];
+        Common.blackhole(a);
+        Common.outerArrayVarHandle.setVolatile(a, index, o);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_LOAD_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_LOAD_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testArrayLoadThenLoad(Outer[] a) {
+        Common.blackhole(Common.outerArrayVarHandle.getVolatile(a, 0));
+        Common.blackhole(Common.outerArrayVarHandle.getVolatile(a, 0));
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testArrayStoreThenStore(Outer[] a, Outer o) {
+        Common.outerArrayVarHandle.setVolatile(a, 0, o);
+        Common.outerArrayVarHandle.setVolatile(a, 0, o);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_LOAD_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testArrayStoreThenLoad(Outer[] a, Outer o) {
+        Common.outerArrayVarHandle.setVolatile(a, 0, o);
+        Common.blackhole(Common.outerArrayVarHandle.getVolatile(a, 0));
+    }
+
+    @Run(test = {"testAllocateArrayThenStoreAtKnownIndex",
+                 "testAllocateArrayThenStoreAtUnknownIndex",
+                 "testArrayLoadThenLoad",
+                 "testArrayStoreThenStore",
+                 "testArrayStoreThenLoad"})
+    void runArrayTests() {
+        testAllocateArrayThenStoreAtKnownIndex(Common.outer);
+        testAllocateArrayThenStoreAtUnknownIndex(Common.outer, 10);
+        testArrayLoadThenLoad(Common.outerArray);
+        testArrayStoreThenStore(Common.outerArray, Common.outer);
+        testArrayStoreThenLoad(Common.outerArray, Common.outer);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testStoreThenConditionalStore(Outer o, Inner i, int value) {
+        o.field1 = i;
+        if (value % 2 == 0) {
+            o.field1 = i;
+        }
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testStoreThenStoreInLoop(Outer o, Inner i) {
+        o.field1 = i;
+        for (int j = 0; j < 100; j++) {
+            o.field1 = i;
+        }
+    }
+
+    @Run(test = {"testStoreThenConditionalStore",
+                 "testStoreThenStoreInLoop"})
+    void runControlFlowTests() {
+        testStoreThenConditionalStore(Common.outer, Common.inner, ThreadLocalRandom.current().nextInt(0, 100));
+        testStoreThenStoreInLoop(Common.outer, Common.inner);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_GET_AND_SET_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testStoreThenAtomic(Outer o, Inner i) {
+        o.field1 = i;
+        Common.field1VarHandle.getAndSet​(o, i);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_GET_AND_SET_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_LOAD_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testAtomicThenLoad(Outer o, Inner i) {
+        Common.field1VarHandle.getAndSet​(o, i);
+        Common.blackhole(o.field1);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_GET_AND_SET_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_STORE_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testAtomicThenStore(Outer o, Inner i) {
+        Common.field1VarHandle.getAndSet​(o, i);
+        o.field1 = i;
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_GET_AND_SET_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_GET_AND_SET_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testAtomicThenAtomic(Outer o, Inner i) {
+        Common.field1VarHandle.getAndSet​(o, i);
+        Common.field1VarHandle.getAndSet​(o, i);
+    }
+
+    @Test
+    @IR(counts = { IRNode.Z_GET_AND_SET_P_WITH_BARRIER_FLAG, Common.REMAINING, "1" }, phase = CompilePhase.FINAL_CODE)
+    @IR(counts = { IRNode.Z_GET_AND_SET_P_WITH_BARRIER_FLAG, Common.ELIDED, "1" }, phase = CompilePhase.FINAL_CODE)
+    static void testArrayAtomicThenAtomic(Outer[] a, Outer o) {
+        Common.outerArrayVarHandle.getAndSet(a, 0, o);
+        Common.outerArrayVarHandle.getAndSet(a, 0, o);
+    }
+
+    @Run(test = {"testStoreThenAtomic",
+                 "testAtomicThenLoad",
+                 "testAtomicThenStore",
+                 "testAtomicThenAtomic",
+                 "testArrayAtomicThenAtomic"})
+    void runAtomicOperationTests() {
+        testStoreThenAtomic(Common.outer, Common.inner);
+        testAtomicThenLoad(Common.outer, Common.inner);
+        testAtomicThenStore(Common.outer, Common.inner);
+        testAtomicThenAtomic(Common.outer, Common.inner);
+        testArrayAtomicThenAtomic(Common.outerArray, Common.outer);
+    }
+}
diff --git a/test/hotspot/jtreg/compiler/gcbarriers/UnsafeIntrinsicsTest.java b/test/hotspot/jtreg/compiler/gcbarriers/UnsafeIntrinsicsTest.java
index c006bc608d1..d39d326ade5 100644
--- a/test/hotspot/jtreg/compiler/gcbarriers/UnsafeIntrinsicsTest.java
+++ b/test/hotspot/jtreg/compiler/gcbarriers/UnsafeIntrinsicsTest.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -21,17 +21,33 @@
  * questions.
  */
 
+/*
+ * @test id=ZDebug
+ * @key randomness
+ * @bug 8059022 8271855
+ * @modules java.base/jdk.internal.misc:+open
+ * @summary Validate barriers after Unsafe getReference, CAS and swap (GetAndSet)
+ * @requires vm.gc.Z & vm.debug
+ * @library /test/lib
+ * @run main/othervm -XX:+UseZGC
+ *                   -XX:+UnlockDiagnosticVMOptions
+ *                   -XX:+ZVerifyOops -XX:ZCollectionInterval=1
+ *                   -XX:-CreateCoredumpOnCrash
+ *                   -XX:CompileCommand=dontinline,*::mergeImpl*
+ *                   compiler.gcbarriers.UnsafeIntrinsicsTest
+ */
+
 /*
  * @test id=Z
  * @key randomness
  * @bug 8059022 8271855
  * @modules java.base/jdk.internal.misc:+open
  * @summary Validate barriers after Unsafe getReference, CAS and swap (GetAndSet)
- * @requires vm.gc.Z
+ * @requires vm.gc.Z & !vm.debug
  * @library /test/lib
  * @run main/othervm -XX:+UseZGC
  *                   -XX:+UnlockDiagnosticVMOptions
- *                   -XX:+ZVerifyViews -XX:ZCollectionInterval=1
+ *                   -XX:ZCollectionInterval=1
  *                   -XX:-CreateCoredumpOnCrash
  *                   -XX:CompileCommand=dontinline,*::mergeImpl*
  *                   compiler.gcbarriers.UnsafeIntrinsicsTest
@@ -289,7 +305,7 @@ class Runner implements Runnable {
     private Node mergeImplLoad(Node startNode, Node expectedNext, Node head) {
         // Atomic load version
         Node temp = (Node) UNSAFE.getReference(startNode, offset);
-        UNSAFE.storeFence(); // Make all new Node fields visible to concurrent readers.
+        UNSAFE.storeFence(); // We need the contents of the published node to be released
         startNode.setNext(head);
         return temp;
     }
diff --git a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java
index 127e96f97d7..def34bb4ac6 100644
--- a/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java
+++ b/test/hotspot/jtreg/compiler/lib/ir_framework/IRNode.java
@@ -1518,6 +1518,24 @@ public class IRNode {
         beforeMatchingNameRegex(EXPAND_BITSV, "ExpandBitsV");
     }
 
+    public static final String Z_LOAD_P_WITH_BARRIER_FLAG = COMPOSITE_PREFIX + "Z_LOAD_P_WITH_BARRIER_FLAG" + POSTFIX;
+    static {
+        String regex = START + "zLoadP\\S*" + MID + "barrier\\(\\s*" + IS_REPLACED + "\\s*\\)" + END;
+        machOnly(Z_LOAD_P_WITH_BARRIER_FLAG, regex);
+    }
+
+    public static final String Z_STORE_P_WITH_BARRIER_FLAG = COMPOSITE_PREFIX + "Z_STORE_P_WITH_BARRIER_FLAG" + POSTFIX;
+    static {
+        String regex = START + "zStoreP\\S*" + MID + "barrier\\(\\s*" + IS_REPLACED + "\\s*\\)" + END;
+        machOnly(Z_STORE_P_WITH_BARRIER_FLAG, regex);
+    }
+
+    public static final String Z_GET_AND_SET_P_WITH_BARRIER_FLAG = COMPOSITE_PREFIX + "Z_GET_AND_SET_P_WITH_BARRIER_FLAG" + POSTFIX;
+    static {
+        String regex = START + "(zXChgP)|(zGetAndSetP\\S*)" + MID + "barrier\\(\\s*" + IS_REPLACED + "\\s*\\)" + END;
+        machOnly(Z_GET_AND_SET_P_WITH_BARRIER_FLAG, regex);
+    }
+
     /*
      * Utility methods to set up IR_NODE_MAPPINGS.
      */
diff --git a/test/hotspot/jtreg/gc/TestSystemGC.java b/test/hotspot/jtreg/gc/TestSystemGC.java
index 9cef6bbdb01..182d41bdc59 100644
--- a/test/hotspot/jtreg/gc/TestSystemGC.java
+++ b/test/hotspot/jtreg/gc/TestSystemGC.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -58,12 +58,21 @@ package gc;
  */
 
 /*
- * @test id=Z
- * @requires vm.gc.Z
+ * @test id=ZSingleGenMode
+ * @requires vm.gc.Z & !vm.opt.final.ZGenerational
  * @comment ZGC will not start when LargePages cannot be allocated, therefore
  *          we do not run such configuration.
  * @summary Runs System.gc() with different flags.
- * @run main/othervm -XX:+UseZGC gc.TestSystemGC
+ * @run main/othervm -XX:+UseZGC -XX:-ZGenerational gc.TestSystemGC
+ */
+
+/*
+ * @test id=Z
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational
+ * @comment ZGC will not start when LargePages cannot be allocated, therefore
+ *          we do not run such configuration.
+ * @summary Runs System.gc() with different flags.
+ * @run main/othervm -XX:+UseZGC -XX:+ZGenerational gc.TestSystemGC
  */
 
 public class TestSystemGC {
diff --git a/test/hotspot/jtreg/gc/TestVerifySubSet.java b/test/hotspot/jtreg/gc/TestVerifySubSet.java
index 6a7c6e86bda..b326dd7fe7f 100644
--- a/test/hotspot/jtreg/gc/TestVerifySubSet.java
+++ b/test/hotspot/jtreg/gc/TestVerifySubSet.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -26,10 +26,10 @@ package gc;
 /* @test TestVerifySubSet.java
  * @bug 8072725
  * @summary Test VerifySubSet option
- *
- * @comment ZGC doesn't use Universe::verify and will not log the output below
- * @requires !vm.gc.Z
- *
+ * @comment Generational ZGC can't use the generic Universe::verify
+ *          because there's no guarantee that we will ever have
+ *          a stable snapshot where all roots can be verified.
+ * @requires vm.gc != "Z"
  * @library /test/lib
  * @modules java.base/jdk.internal.misc
  * @run main gc.TestVerifySubSet
diff --git a/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithZ.java b/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithZ.java
index 4a8e1f2019a..66009562f9d 100644
--- a/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithZ.java
+++ b/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithZ.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2016, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -30,20 +30,41 @@ import java.io.IOException;
  * @test TestGCBasherWithZ
  * @key stress
  * @library /
- * @requires vm.gc.Z
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational
  * @requires vm.flavor == "server" & !vm.emulatedClient
  * @summary Stress ZGC
- * @run main/othervm/timeout=200 -Xlog:gc*=info -Xmx384m -server -XX:+UseZGC gc.stress.gcbasher.TestGCBasherWithZ 120000
+ * @run main/othervm/timeout=200 -Xlog:gc*=info -Xmx384m -server -XX:+UseZGC -XX:+ZGenerational  gc.stress.gcbasher.TestGCBasherWithZ 120000
+ */
+/*
+ * @test TestGCBasherWithZSingleGenMode
+ * @key stress
+ * @library /
+ * @requires vm.gc.Z & !vm.opt.final.ZGenerational
+ * @requires vm.flavor == "server" & !vm.emulatedClient
+ * @summary Stress ZGC
+ * @run main/othervm/timeout=200 -Xlog:gc*=info -Xmx384m -server -XX:+UseZGC -XX:-ZGenerational gc.stress.gcbasher.TestGCBasherWithZ 120000
  */
 
 /*
  * @test TestGCBasherDeoptWithZ
  * @key stress
  * @library /
- * @requires vm.gc.Z
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational
  * @requires vm.flavor == "server" & !vm.emulatedClient & vm.opt.ClassUnloading != false
  * @summary Stress ZGC with nmethod barrier forced deoptimization enabled.
- * @run main/othervm/timeout=200 -Xlog:gc*=info,nmethod+barrier=trace -Xmx384m -server -XX:+UseZGC
+ * @run main/othervm/timeout=200 -Xlog:gc*=info,nmethod+barrier=trace -Xmx384m -server -XX:+UseZGC -XX:+ZGenerational
+ *   -XX:+UnlockDiagnosticVMOptions -XX:+DeoptimizeNMethodBarriersALot -XX:-Inline
+ *   gc.stress.gcbasher.TestGCBasherWithZ 120000
+ */
+
+/*
+ * @test TestGCBasherDeoptWithZSingleGenMode
+ * @key stress
+ * @library /
+ * @requires vm.gc.Z & !vm.opt.final.ZGenerational
+ * @requires vm.flavor == "server" & !vm.emulatedClient & vm.opt.ClassUnloading != false
+ * @summary Stress ZGC with nmethod barrier forced deoptimization enabled.
+ * @run main/othervm/timeout=200 -Xlog:gc*=info,nmethod+barrier=trace -Xmx384m -server -XX:+UseZGC -XX:-ZGenerational
  *   -XX:+UnlockDiagnosticVMOptions -XX:+DeoptimizeNMethodBarriersALot -XX:-Inline
  *   gc.stress.gcbasher.TestGCBasherWithZ 120000
  */
diff --git a/test/hotspot/jtreg/gc/stress/gcold/TestGCOldWithZ.java b/test/hotspot/jtreg/gc/stress/gcold/TestGCOldWithZ.java
index 01998ef0c90..46ce8b7f548 100644
--- a/test/hotspot/jtreg/gc/stress/gcold/TestGCOldWithZ.java
+++ b/test/hotspot/jtreg/gc/stress/gcold/TestGCOldWithZ.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -28,10 +28,20 @@ package gc.stress.gcold;
  * @test TestGCOldWithZ
  * @key randomness
  * @library / /test/lib
- * @requires vm.gc.Z
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational
  * @summary Stress the Z
- * @run main/othervm -Xmx384M -XX:+UseZGC gc.stress.gcold.TestGCOldWithZ 50 1 20 10 10000
- * @run main/othervm -Xmx256m -XX:+UseZGC gc.stress.gcold.TestGCOldWithZ 50 5 20 1 5000
+ * @run main/othervm -Xmx384M -XX:+UseZGC -XX:+ZGenerational gc.stress.gcold.TestGCOldWithZ 50 1 20 10 10000
+ * @run main/othervm -Xmx256m -XX:+UseZGC -XX:+ZGenerational gc.stress.gcold.TestGCOldWithZ 50 5 20 1 5000
+ */
+
+/*
+ * @test TestGCOldWithZSingleGenMode
+ * @key randomness
+ * @library / /test/lib
+ * @requires vm.gc.Z & !vm.opt.final.ZGenerational
+ * @summary Stress the Z
+ * @run main/othervm -Xmx384M -XX:+UseZGC -XX:-ZGenerational gc.stress.gcold.TestGCOldWithZ 50 1 20 10 10000
+ * @run main/othervm -Xmx256m -XX:+UseZGC -XX:-ZGenerational gc.stress.gcold.TestGCOldWithZ 50 5 20 1 5000
  */
 public class TestGCOldWithZ {
     public static void main(String[] args) {
diff --git a/test/hotspot/jtreg/gc/stringdedup/TestStringDeduplicationTools.java b/test/hotspot/jtreg/gc/stringdedup/TestStringDeduplicationTools.java
index f0f93676251..ac2ead65ac1 100644
--- a/test/hotspot/jtreg/gc/stringdedup/TestStringDeduplicationTools.java
+++ b/test/hotspot/jtreg/gc/stringdedup/TestStringDeduplicationTools.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -100,10 +100,21 @@ class TestStringDeduplicationTools {
             if (n.getType().equals(GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION)) {
                 GarbageCollectionNotificationInfo info = GarbageCollectionNotificationInfo.from((CompositeData) n.getUserData());
                 // Shenandoah and Z GC also report GC pauses, skip them
-                if (info.getGcName().startsWith("Shenandoah") || info.getGcName().startsWith("ZGC")) {
+                if (info.getGcName().startsWith("Shenandoah")) {
                     if ("end of GC cycle".equals(info.getGcAction())) {
                         gcCount++;
                     }
+                } else if (info.getGcName().startsWith("ZGC")) {
+                    // Generational ZGC only triggers string deduplications from major collections
+                    if (info.getGcName().startsWith("ZGC Major") && "end of GC cycle".equals(info.getGcAction())) {
+                        gcCount++;
+                    }
+
+                    // Single-gen ZGC
+                    if (!info.getGcName().startsWith("ZGC Major") && !info.getGcName().startsWith("ZGC Minor") &&
+                            "end of GC cycle".equals(info.getGcAction())) {
+                        gcCount++;
+                    }
                 } else if (info.getGcName().startsWith("G1")) {
                     if ("end of minor GC".equals(info.getGcAction())) {
                         gcCount++;
diff --git a/test/hotspot/jtreg/gc/x/TestAllocateHeapAt.java b/test/hotspot/jtreg/gc/x/TestAllocateHeapAt.java
new file mode 100644
index 00000000000..cebe321b2cb
--- /dev/null
+++ b/test/hotspot/jtreg/gc/x/TestAllocateHeapAt.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.x;
+
+/*
+ * @test TestAllocateHeapAt
+ * @requires vm.gc.Z & !vm.opt.final.ZGenerational & os.family == "linux"
+ * @summary Test ZGC with -XX:AllocateHeapAt
+ * @library /test/lib
+ * @run main/othervm gc.x.TestAllocateHeapAt . true
+ * @run main/othervm gc.x.TestAllocateHeapAt non-existing-directory false
+ */
+
+import jdk.test.lib.process.ProcessTools;
+
+public class TestAllocateHeapAt {
+    public static void main(String[] args) throws Exception {
+        final String directory = args[0];
+        final boolean exists = Boolean.parseBoolean(args[1]);
+        final String heapBackingFile = "Heap Backing File: " + directory;
+        final String failedToCreateFile = "Failed to create file " + directory;
+
+        ProcessTools.executeProcess(ProcessTools.createJavaProcessBuilder(
+                "-XX:+UseZGC",
+                "-XX:-ZGenerational",
+                "-Xlog:gc*",
+                "-Xms32M",
+                "-Xmx32M",
+                "-XX:AllocateHeapAt=" + directory,
+                "-version"))
+            .shouldContain(exists ? heapBackingFile : failedToCreateFile)
+            .shouldNotContain(exists ? failedToCreateFile : heapBackingFile)
+            .shouldHaveExitValue(exists ? 0 : 1);
+    }
+}
diff --git a/test/hotspot/jtreg/gc/x/TestAlwaysPreTouch.java b/test/hotspot/jtreg/gc/x/TestAlwaysPreTouch.java
new file mode 100644
index 00000000000..af6bf9d3f80
--- /dev/null
+++ b/test/hotspot/jtreg/gc/x/TestAlwaysPreTouch.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.x;
+
+/*
+ * @test TestAlwaysPreTouch
+ * @requires vm.gc.Z & !vm.opt.final.ZGenerational
+ * @summary Test ZGC parallel pre-touch
+ * @run main/othervm -XX:+UseZGC -XX:-ZGenerational -Xlog:gc* -XX:-AlwaysPreTouch -Xms128M -Xmx128M gc.x.TestAlwaysPreTouch
+ * @run main/othervm -XX:+UseZGC -XX:-ZGenerational -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=1 -Xms2M -Xmx128M gc.x.TestAlwaysPreTouch
+ * @run main/othervm -XX:+UseZGC -XX:-ZGenerational -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=8 -Xms2M -Xmx128M gc.x.TestAlwaysPreTouch
+ * @run main/othervm -XX:+UseZGC -XX:-ZGenerational -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=1 -Xms128M -Xmx128M gc.x.TestAlwaysPreTouch
+ * @run main/othervm -XX:+UseZGC -XX:-ZGenerational -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=8 -Xms128M -Xmx128M gc.x.TestAlwaysPreTouch
+ */
+
+public class TestAlwaysPreTouch {
+    public static void main(String[] args) throws Exception {
+        System.out.println("Success");
+    }
+}
diff --git a/test/hotspot/jtreg/gc/x/TestGarbageCollectorMXBean.java b/test/hotspot/jtreg/gc/x/TestGarbageCollectorMXBean.java
new file mode 100644
index 00000000000..2cb72053afa
--- /dev/null
+++ b/test/hotspot/jtreg/gc/x/TestGarbageCollectorMXBean.java
@@ -0,0 +1,220 @@
+/*
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.x;
+
+/**
+ * @test TestGarbageCollectorMXBean
+ * @requires vm.gc.Z & !vm.opt.final.ZGenerational
+ * @summary Test ZGC garbage collector MXBean
+ * @modules java.management
+ * @run main/othervm -XX:+UseZGC -XX:-ZGenerational -Xms256M -Xmx512M -Xlog:gc gc.x.TestGarbageCollectorMXBean 256 512
+ * @run main/othervm -XX:+UseZGC -XX:-ZGenerational -Xms512M -Xmx512M -Xlog:gc gc.x.TestGarbageCollectorMXBean 512 512
+ */
+
+import java.lang.management.ManagementFactory;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+import javax.management.Notification;
+import javax.management.NotificationEmitter;
+import javax.management.NotificationListener;
+import javax.management.openmbean.CompositeData;
+
+import com.sun.management.GarbageCollectionNotificationInfo;
+
+public class TestGarbageCollectorMXBean {
+    private static final long startTime = System.nanoTime();
+
+    private static void log(String msg) {
+        final String elapsedSeconds = String.format("%.3fs", (System.nanoTime() - startTime) / 1_000_000_000.0);
+        System.out.println("[" + elapsedSeconds + "] (" + Thread.currentThread().getName() + ") " + msg);
+    }
+
+    public static void main(String[] args) throws Exception {
+        final long M = 1024 * 1024;
+        final long initialCapacity = Long.parseLong(args[0]) * M;
+        final long maxCapacity = Long.parseLong(args[1]) * M;
+        final AtomicInteger cycles = new AtomicInteger();
+        final AtomicInteger pauses = new AtomicInteger();
+        final AtomicInteger errors = new AtomicInteger();
+
+        final NotificationListener listener = (Notification notification, Object ignored) -> {
+            final var type = notification.getType();
+            if (!type.equals(GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION)) {
+                // Ignore
+                return;
+            }
+
+            final var data = (CompositeData)notification.getUserData();
+            final var info = GarbageCollectionNotificationInfo.from(data);
+            final var name = info.getGcName();
+            final var id = info.getGcInfo().getId();
+            final var action = info.getGcAction();
+            final var cause = info.getGcCause();
+            final var startTime = info.getGcInfo().getStartTime();
+            final var endTime = info.getGcInfo().getEndTime();
+            final var duration = info.getGcInfo().getDuration();
+            final var memoryUsageBeforeGC = info.getGcInfo().getMemoryUsageBeforeGc().get("ZHeap");
+            final var memoryUsageAfterGC = info.getGcInfo().getMemoryUsageAfterGc().get("ZHeap");
+
+            log(name + " (" + type + ")");
+            log("                  Id: " + id);
+            log("              Action: " + action);
+            log("               Cause: " + cause);
+            log("           StartTime: " + startTime);
+            log("             EndTime: " + endTime);
+            log("            Duration: " + duration);
+            log(" MemoryUsageBeforeGC: " + memoryUsageBeforeGC);
+            log("  MemoryUsageAfterGC: " + memoryUsageAfterGC);
+            log("");
+
+            if (name.equals("ZGC Cycles")) {
+                cycles.incrementAndGet();
+
+                if (!action.equals("end of GC cycle")) {
+                    log("ERROR: Action");
+                    errors.incrementAndGet();
+                }
+
+                if (memoryUsageBeforeGC.getInit() != initialCapacity) {
+                    log("ERROR: MemoryUsageBeforeGC.init");
+                    errors.incrementAndGet();
+                }
+
+                if (memoryUsageBeforeGC.getUsed() > initialCapacity) {
+                    log("ERROR: MemoryUsageBeforeGC.used");
+                    errors.incrementAndGet();
+                }
+
+                if (memoryUsageBeforeGC.getCommitted() != initialCapacity) {
+                    log("ERROR: MemoryUsageBeforeGC.committed");
+                    errors.incrementAndGet();
+                }
+
+                if (memoryUsageBeforeGC.getMax() != maxCapacity) {
+                    log("ERROR: MemoryUsageBeforeGC.max");
+                    errors.incrementAndGet();
+                }
+            } else if (name.equals("ZGC Pauses")) {
+                pauses.incrementAndGet();
+
+                if (!action.equals("end of GC pause")) {
+                    log("ERROR: Action");
+                    errors.incrementAndGet();
+                }
+
+                if (memoryUsageBeforeGC.getInit() != 0) {
+                    log("ERROR: MemoryUsageBeforeGC.init");
+                    errors.incrementAndGet();
+                }
+
+                if (memoryUsageBeforeGC.getUsed() != 0) {
+                    log("ERROR: MemoryUsageBeforeGC.used");
+                    errors.incrementAndGet();
+                }
+
+                if (memoryUsageBeforeGC.getCommitted() != 0) {
+                    log("ERROR: MemoryUsageBeforeGC.committed");
+                    errors.incrementAndGet();
+                }
+
+                if (memoryUsageBeforeGC.getMax() != 0) {
+                    log("ERROR: MemoryUsageBeforeGC.max");
+                    errors.incrementAndGet();
+                }
+            } else {
+                log("ERROR: Name");
+                errors.incrementAndGet();
+            }
+
+            if (!cause.equals("System.gc()")) {
+                log("ERROR: Cause");
+                errors.incrementAndGet();
+            }
+
+            if (startTime > endTime) {
+                log("ERROR: StartTime");
+                errors.incrementAndGet();
+            }
+
+            if (endTime - startTime != duration) {
+                log("ERROR: Duration");
+                errors.incrementAndGet();
+            }
+        };
+
+        // Collect garbage created at startup
+        System.gc();
+
+        // Register GC event listener
+        for (final var collector : ManagementFactory.getGarbageCollectorMXBeans()) {
+            final NotificationEmitter emitter = (NotificationEmitter)collector;
+            emitter.addNotificationListener(listener, null, null);
+        }
+
+        final int minCycles = 5;
+        final int minPauses = minCycles * 3;
+
+        // Run GCs
+        for (int i = 0; i < minCycles; i++) {
+            log("Starting GC " + i);
+            System.gc();
+        }
+
+        // Wait at most 90 seconds
+        for (int i = 0; i < 90; i++) {
+            log("Waiting...");
+            Thread.sleep(1000);
+
+            if (cycles.get() >= minCycles) {
+                log("All events received!");
+                break;
+            }
+        }
+
+        final int actualCycles = cycles.get();
+        final int actualPauses = pauses.get();
+        final int actualErrors = errors.get();
+
+        log("   minCycles: " + minCycles);
+        log("   minPauses: " + minPauses);
+        log("actualCycles: " + actualCycles);
+        log("actualPauses: " + actualPauses);
+        log("actualErrors: " + actualErrors);
+
+        // Verify number of cycle events
+        if (actualCycles < minCycles) {
+            throw new Exception("Unexpected cycles");
+        }
+
+        // Verify number of pause events
+        if (actualPauses < minPauses) {
+            throw new Exception("Unexpected pauses");
+        }
+
+        // Verify number of errors
+        if (actualErrors != 0) {
+            throw new Exception("Unexpected errors");
+        }
+    }
+}
diff --git a/test/hotspot/jtreg/gc/x/TestHighUsage.java b/test/hotspot/jtreg/gc/x/TestHighUsage.java
new file mode 100644
index 00000000000..d6125fb35a2
--- /dev/null
+++ b/test/hotspot/jtreg/gc/x/TestHighUsage.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.x;
+
+/*
+ * @test TestHighUsage
+ * @requires vm.gc.Z & !vm.opt.final.ZGenerational
+ * @summary Test ZGC "High Usage" rule
+ * @library /test/lib
+ * @run main/othervm gc.x.TestHighUsage
+ */
+
+import java.util.LinkedList;
+import jdk.test.lib.process.ProcessTools;
+
+public class TestHighUsage {
+    static class Test {
+        private static final int K = 1024;
+        private static final int M = K * K;
+        private static final long maxCapacity = Runtime.getRuntime().maxMemory();
+        private static final long slowAllocationThreshold = 16 * M;
+        private static final long highUsageThreshold = maxCapacity / 20; // 5%
+        private static volatile LinkedList<byte[]> keepAlive;
+        private static volatile Object dummy;
+
+        public static void main(String[] args) throws Exception {
+            System.out.println("Max capacity: " + (maxCapacity / M) + "M");
+            System.out.println("High usage threshold: " + (highUsageThreshold / M) + "M");
+            System.out.println("Allocating live-set");
+
+            // Allocate live-set
+            keepAlive = new LinkedList<>();
+            while (Runtime.getRuntime().freeMemory() > slowAllocationThreshold) {
+                while (Runtime.getRuntime().freeMemory() > slowAllocationThreshold) {
+                    keepAlive.add(new byte[128 * K]);
+                }
+
+                // Compact live-set and let allocation rate settle down
+                System.gc();
+                Thread.sleep(2000);
+            }
+
+            System.out.println("Allocating garbage slowly");
+
+            // Allocate garbage slowly, so that the sampled allocation rate on average
+            // becomes zero MB/s for the last 1 second windows. Once we reach the high
+            // usage threshold we idle to allow for a "High Usage" GC cycle to happen.
+            // We need to allocate slowly to avoid an "Allocation Rate" GC cycle.
+            for (int i = 0; i < 300; i++) {
+                if (Runtime.getRuntime().freeMemory() > highUsageThreshold) {
+                    // Allocate
+                    dummy = new byte[128 * K];
+                    System.out.println("Free: " + (Runtime.getRuntime().freeMemory() / M) + "M (Allocating)");
+                } else {
+                    // Idle
+                    System.out.println("Free: " + (Runtime.getRuntime().freeMemory() / M) + "M (Idling)");
+                }
+
+                Thread.sleep(250);
+            }
+
+            System.out.println("Done");
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        ProcessTools.executeTestJvm("-XX:+UseZGC",
+                                    "-XX:-ZGenerational",
+                                    "-XX:-ZProactive",
+                                    "-Xms128M",
+                                    "-Xmx128M",
+                                    "-XX:ParallelGCThreads=1",
+                                    "-XX:ConcGCThreads=1",
+                                    "-Xlog:gc,gc+start",
+                                    Test.class.getName())
+                    .shouldNotContain("Allocation Stall")
+                    .shouldContain("High Usage")
+                    .shouldHaveExitValue(0);
+    }
+}
diff --git a/test/hotspot/jtreg/gc/x/TestMemoryMXBean.java b/test/hotspot/jtreg/gc/x/TestMemoryMXBean.java
new file mode 100644
index 00000000000..a4a0a8544d6
--- /dev/null
+++ b/test/hotspot/jtreg/gc/x/TestMemoryMXBean.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.x;
+
+/**
+ * @test TestMemoryMXBean
+ * @requires vm.gc.Z & !vm.opt.final.ZGenerational
+ * @summary Test ZGC heap memory MXBean
+ * @modules java.management
+ * @run main/othervm -XX:+UseZGC -XX:-ZGenerational -Xms128M -Xmx256M -Xlog:gc* gc.x.TestMemoryMXBean 128 256
+ * @run main/othervm -XX:+UseZGC -XX:-ZGenerational -Xms256M -Xmx256M -Xlog:gc* gc.x.TestMemoryMXBean 256 256
+ */
+
+import java.lang.management.ManagementFactory;
+
+public class TestMemoryMXBean {
+    public static void main(String[] args) throws Exception {
+        final long M = 1024 * 1024;
+        final long expectedInitialCapacity = Long.parseLong(args[0]) * M;
+        final long expectedMaxCapacity = Long.parseLong(args[1]) * M;
+        final var memoryUsage = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage();
+        final long initialCapacity = memoryUsage.getInit();
+        final long capacity = memoryUsage.getCommitted();
+        final long maxCapacity = memoryUsage.getMax();
+
+        System.out.println("expectedInitialCapacity: " + expectedInitialCapacity);
+        System.out.println("    expectedMaxCapacity: " + expectedMaxCapacity);
+        System.out.println("        initialCapacity: " + initialCapacity);
+        System.out.println("               capacity: " + capacity);
+        System.out.println("            maxCapacity: " + maxCapacity);
+
+        if (initialCapacity != expectedInitialCapacity) {
+            throw new Exception("Unexpected initial capacity");
+        }
+
+        if (maxCapacity != expectedMaxCapacity) {
+            throw new Exception("Unexpected max capacity");
+        }
+
+        if (capacity < initialCapacity || capacity > maxCapacity) {
+            throw new Exception("Unexpected capacity");
+        }
+    }
+}
diff --git a/test/hotspot/jtreg/gc/x/TestMemoryManagerMXBean.java b/test/hotspot/jtreg/gc/x/TestMemoryManagerMXBean.java
new file mode 100644
index 00000000000..b219cad7211
--- /dev/null
+++ b/test/hotspot/jtreg/gc/x/TestMemoryManagerMXBean.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.x;
+
+/**
+ * @test TestMemoryManagerMXBean
+ * @requires vm.gc.Z & !vm.opt.final.ZGenerational
+ * @summary Test ZGC memory manager MXBean
+ * @modules java.management
+ * @run main/othervm -XX:+UseZGC -XX:-ZGenerational -Xmx128M gc.x.TestMemoryManagerMXBean
+ */
+
+import java.lang.management.ManagementFactory;
+
+public class TestMemoryManagerMXBean {
+    private static void checkName(String name) throws Exception {
+        if (name == null || name.length() == 0) {
+            throw new Exception("Invalid name");
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        int zgcCyclesMemoryManagers = 0;
+        int zgcPausesMemoryManagers = 0;
+        int zgcCyclesMemoryPools = 0;
+        int zgcPausesMemoryPools = 0;
+
+        for (final var memoryManager : ManagementFactory.getMemoryManagerMXBeans()) {
+            final var memoryManagerName = memoryManager.getName();
+            checkName(memoryManagerName);
+
+            System.out.println("MemoryManager: " + memoryManagerName);
+
+            if (memoryManagerName.equals("ZGC Cycles")) {
+                zgcCyclesMemoryManagers++;
+            } else if (memoryManagerName.equals("ZGC Pauses")) {
+                zgcPausesMemoryManagers++;
+            }
+
+            for (final var memoryPoolName : memoryManager.getMemoryPoolNames()) {
+                checkName(memoryPoolName);
+
+                System.out.println("   MemoryPool:   " + memoryPoolName);
+
+                if (memoryPoolName.equals("ZHeap")) {
+                    if (memoryManagerName.equals("ZGC Cycles")) {
+                        zgcCyclesMemoryPools++;
+                    } else if (memoryManagerName.equals("ZGC Pauses")) {
+                        zgcPausesMemoryPools++;
+                    }
+                }
+            }
+        }
+
+        if (zgcCyclesMemoryManagers != 1) {
+            throw new Exception("Unexpected number of cycle MemoryManagers");
+        }
+
+        if (zgcPausesMemoryManagers != 1) {
+            throw new Exception("Unexpected number of pause MemoryManagers");
+        }
+
+        if (zgcCyclesMemoryPools != 1) {
+            throw new Exception("Unexpected number of cycle MemoryPools");
+        }
+
+        if (zgcPausesMemoryPools != 1) {
+            throw new Exception("Unexpected number of pause MemoryPools");
+        }
+    }
+}
diff --git a/test/hotspot/jtreg/gc/x/TestNoUncommit.java b/test/hotspot/jtreg/gc/x/TestNoUncommit.java
new file mode 100644
index 00000000000..089cdaebe0b
--- /dev/null
+++ b/test/hotspot/jtreg/gc/x/TestNoUncommit.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.x;
+
+/*
+ * @test TestNoUncommit
+ * @requires vm.gc.Z  & !vm.opt.final.ZGenerational & !vm.graal.enabled
+ * @summary Test ZGC uncommit unused memory disabled
+ * @run main/othervm -XX:+UseZGC -XX:-ZGenerational -Xlog:gc*,gc+heap=debug,gc+stats=off -Xms512M -Xmx512M -XX:ZUncommitDelay=1 gc.x.TestNoUncommit
+ * @run main/othervm -XX:+UseZGC -XX:-ZGenerational -Xlog:gc*,gc+heap=debug,gc+stats=off -Xms128M -Xmx512M -XX:ZUncommitDelay=1 -XX:-ZUncommit gc.x.TestNoUncommit
+ */
+
+public class TestNoUncommit {
+    private static final int allocSize = 200 * 1024 * 1024; // 200M
+    private static volatile Object keepAlive = null;
+
+    private static long capacity() {
+        return Runtime.getRuntime().totalMemory();
+    }
+
+    public static void main(String[] args) throws Exception {
+        System.out.println("Allocating");
+        keepAlive = new byte[allocSize];
+        final var afterAlloc = capacity();
+
+        System.out.println("Reclaiming");
+        keepAlive = null;
+        System.gc();
+
+        // Wait longer than the uncommit delay (which is 1 second)
+        Thread.sleep(5 * 1000);
+
+        final var afterDelay = capacity();
+
+        // Verify
+        if (afterAlloc > afterDelay) {
+            throw new Exception("Should not uncommit");
+        }
+
+        System.out.println("Success");
+    }
+}
diff --git a/test/hotspot/jtreg/gc/x/TestPageCacheFlush.java b/test/hotspot/jtreg/gc/x/TestPageCacheFlush.java
new file mode 100644
index 00000000000..96790d6cb29
--- /dev/null
+++ b/test/hotspot/jtreg/gc/x/TestPageCacheFlush.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.x;
+
+/*
+ * @test TestPageCacheFlush
+ * @requires vm.gc.Z & !vm.opt.final.ZGenerational
+ * @summary Test ZGC page cache flushing
+ * @library /test/lib
+ * @run driver gc.x.TestPageCacheFlush
+ */
+
+import java.util.LinkedList;
+import jdk.test.lib.process.ProcessTools;
+
+public class TestPageCacheFlush {
+    static class Test {
+        private static final int K = 1024;
+        private static final int M = K * K;
+        private static volatile LinkedList<byte[]> keepAlive;
+
+        public static void fillPageCache(int size) {
+            System.out.println("Begin allocate (" + size + ")");
+
+            keepAlive = new LinkedList<>();
+
+            try {
+                for (;;) {
+                    keepAlive.add(new byte[size]);
+                }
+            } catch (OutOfMemoryError e) {
+                keepAlive = null;
+                System.gc();
+            }
+
+            System.out.println("End allocate (" + size + ")");
+        }
+
+        public static void main(String[] args) throws Exception {
+            // Allocate small objects to fill the page cache with small pages
+            fillPageCache(10 * K);
+
+            // Allocate large objects to provoke page cache flushing to rebuild
+            // cached small pages into large pages
+            fillPageCache(10 * M);
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        ProcessTools.executeProcess(ProcessTools.createJavaProcessBuilder(
+                                    "-XX:+UseZGC",
+                                    "-XX:-ZGenerational",
+                                    "-Xms128M",
+                                    "-Xmx128M",
+                                    "-Xlog:gc,gc+init,gc+heap=debug",
+                                    Test.class.getName()))
+                    .outputTo(System.out)
+                    .errorTo(System.out)
+                    .shouldContain("Page Cache Flushed:")
+                    .shouldHaveExitValue(0);
+    }
+}
diff --git a/test/hotspot/jtreg/gc/x/TestRelocateInPlace.java b/test/hotspot/jtreg/gc/x/TestRelocateInPlace.java
new file mode 100644
index 00000000000..97f72451cd6
--- /dev/null
+++ b/test/hotspot/jtreg/gc/x/TestRelocateInPlace.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.x;
+
+/*
+ * @test TestRelocateInPlace
+ * @requires vm.gc.Z & !vm.opt.final.ZGenerational
+ * @summary Test ZGC in-place relocateion
+ * @run main/othervm -XX:+UseZGC -XX:-ZGenerational -Xlog:gc*,gc+stats=off -Xmx256M -XX:+UnlockDiagnosticVMOptions -XX:+ZStressRelocateInPlace gc.x.TestRelocateInPlace
+ */
+
+import java.util.ArrayList;
+
+public class TestRelocateInPlace {
+    private static final int allocSize = 100 * 1024 * 1024; // 100M
+    private static final int smallObjectSize = 4 * 1024; // 4K
+    private static final int mediumObjectSize = 2 * 1024 * 1024; // 2M
+
+    private static volatile ArrayList<byte[]> keepAlive;
+
+    private static void allocate(int objectSize) {
+        keepAlive = new ArrayList<>();
+        for (int i = 0; i < allocSize; i+= objectSize) {
+            keepAlive.add(new byte[objectSize]);
+        }
+    }
+
+    private static void fragment() {
+        // Release every other reference to cause lots of fragmentation
+        for (int i = 0; i < keepAlive.size(); i += 2) {
+            keepAlive.set(i, null);
+        }
+    }
+
+    private static void test(int objectSize) throws Exception {
+        System.out.println("Allocating");
+        allocate(objectSize);
+
+        System.out.println("Fragmenting");
+        fragment();
+
+        System.out.println("Reclaiming");
+        System.gc();
+    }
+
+    public static void main(String[] args) throws Exception {
+        for (int i = 0; i < 10; i++) {
+            System.out.println("Iteration " + i);
+            test(smallObjectSize);
+            test(mediumObjectSize);
+        }
+    }
+}
diff --git a/test/hotspot/jtreg/gc/x/TestSmallHeap.java b/test/hotspot/jtreg/gc/x/TestSmallHeap.java
new file mode 100644
index 00000000000..e8d342cd6d8
--- /dev/null
+++ b/test/hotspot/jtreg/gc/x/TestSmallHeap.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.x;
+
+/*
+ * @test TestSmallHeap
+ * @requires vm.gc.Z & !vm.opt.final.ZGenerational
+ * @summary Test ZGC with small heaps
+ * @library / /test/lib
+ * @run driver gc.x.TestSmallHeap 8M 16M 32M 64M 128M 256M 512M 1024M
+ */
+
+import jdk.test.lib.process.ProcessTools;
+import static gc.testlibrary.Allocation.blackHole;
+
+public class TestSmallHeap {
+    public static class Test {
+        public static void main(String[] args) throws Exception {
+            final long maxCapacity = Runtime.getRuntime().maxMemory();
+            System.out.println("Max Capacity " + maxCapacity + " bytes");
+
+            // Allocate byte arrays of increasing length, so that
+            // all allocation paths (small/medium/large) are tested.
+            for (int length = 16; length <= maxCapacity / 16; length *= 2) {
+                System.out.println("Allocating " + length + " bytes");
+                blackHole(new byte[length]);
+            }
+
+            System.out.println("Success");
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        for (var maxCapacity: args) {
+            ProcessTools.executeProcess(ProcessTools.createJavaProcessBuilder(
+                                        "-XX:+UseZGC",
+                                        "-XX:-ZGenerational",
+                                        "-Xlog:gc,gc+init,gc+reloc,gc+heap",
+                                        "-Xmx" + maxCapacity,
+                                        Test.class.getName()))
+                .outputTo(System.out)
+                .errorTo(System.out)
+                .shouldContain("Success")
+                .shouldHaveExitValue(0);
+        }
+    }
+}
diff --git a/test/hotspot/jtreg/gc/x/TestUncommit.java b/test/hotspot/jtreg/gc/x/TestUncommit.java
new file mode 100644
index 00000000000..949b11cebab
--- /dev/null
+++ b/test/hotspot/jtreg/gc/x/TestUncommit.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package gc.x;
+
+/*
+ * @test TestUncommit
+ * @requires vm.gc.Z & !vm.opt.final.ZGenerational
+ * @summary Test ZGC uncommit unused memory
+ * @library /test/lib
+ * @run main/othervm -XX:+UseZGC -XX:-ZGenerational -Xlog:gc*,gc+heap=debug,gc+stats=off -Xms128M -Xmx512M -XX:ZUncommitDelay=10 gc.x.TestUncommit
+ */
+
+import java.util.ArrayList;
+import jdk.test.lib.Utils;
+
+public class TestUncommit {
+    private static final int delay = 10 * 1000; // milliseconds
+    private static final int allocSize = 200 * 1024 * 1024; // 200M
+    private static final int smallObjectSize = 4 * 1024; // 4K
+    private static final int mediumObjectSize = 2 * 1024 * 1024; // 2M
+    private static final int largeObjectSize = allocSize;
+
+    private static volatile ArrayList<byte[]> keepAlive;
+
+    private static final long startTime = System.nanoTime();
+
+    private static void log(String msg) {
+        final String elapsedSeconds = String.format("%.3fs", (System.nanoTime() - startTime) / 1_000_000_000.0);
+        System.out.println("[" + elapsedSeconds + "] (" + Thread.currentThread().getName() + ") " + msg);
+    }
+
+    private static long capacity() {
+        return Runtime.getRuntime().totalMemory();
+    }
+
+    private static void allocate(int objectSize) {
+        keepAlive = new ArrayList<>();
+        for (int i = 0; i < allocSize; i+= objectSize) {
+            keepAlive.add(new byte[objectSize]);
+        }
+    }
+
+    private static void reclaim() {
+        keepAlive = null;
+        System.gc();
+    }
+
+    private static void test(int objectSize) throws Exception {
+        final var beforeAlloc = capacity();
+        final var timeBeforeAlloc = System.nanoTime();
+
+        // Allocate memory
+        log("Allocating");
+        allocate(objectSize);
+
+        final var afterAlloc = capacity();
+
+        // Reclaim memory
+        log("Reclaiming");
+        reclaim();
+
+        log("Waiting for uncommit to start");
+        while (capacity() >= afterAlloc) {
+            Thread.sleep(1000);
+        }
+
+        log("Uncommit started");
+        final var timeUncommitStart = System.nanoTime();
+        final var actualDelay = (timeUncommitStart - timeBeforeAlloc) / 1_000_000;
+
+        log("Waiting for uncommit to complete");
+        while (capacity() > beforeAlloc) {
+            Thread.sleep(1000);
+        }
+
+        log("Uncommit completed");
+        final var afterUncommit = capacity();
+
+        log("        Uncommit Delay: " + delay);
+        log("           Object Size: " + objectSize);
+        log("            Alloc Size: " + allocSize);
+        log("          Before Alloc: " + beforeAlloc);
+        log("           After Alloc: " + afterAlloc);
+        log("        After Uncommit: " + afterUncommit);
+        log(" Actual Uncommit Delay: " + actualDelay);
+
+        // Verify
+        if (actualDelay < delay) {
+            throw new Exception("Uncommitted too fast");
+        }
+
+        if (actualDelay > delay * 2 * Utils.TIMEOUT_FACTOR) {
+            throw new Exception("Uncommitted too slow");
+        }
+
+        if (afterUncommit < beforeAlloc) {
+            throw new Exception("Uncommitted too much");
+        }
+
+        if (afterUncommit > beforeAlloc) {
+            throw new Exception("Uncommitted too little");
+        }
+
+        log("Success");
+    }
+
+    public static void main(String[] args) throws Exception {
+        for (int i = 0; i < 2; i++) {
+            log("Iteration " + i);
+            test(smallObjectSize);
+            test(mediumObjectSize);
+            test(largeObjectSize);
+        }
+    }
+}
diff --git a/test/hotspot/jtreg/gc/z/TestAllocateHeapAt.java b/test/hotspot/jtreg/gc/z/TestAllocateHeapAt.java
index eed7426f791..f543c5a14d5 100644
--- a/test/hotspot/jtreg/gc/z/TestAllocateHeapAt.java
+++ b/test/hotspot/jtreg/gc/z/TestAllocateHeapAt.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,7 @@ package gc.z;
 
 /*
  * @test TestAllocateHeapAt
- * @requires vm.gc.Z & os.family == "linux"
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational & os.family == "linux"
  * @summary Test ZGC with -XX:AllocateHeapAt
  * @library /test/lib
  * @run main/othervm gc.z.TestAllocateHeapAt . true
@@ -43,6 +43,7 @@ public class TestAllocateHeapAt {
 
         ProcessTools.executeProcess(ProcessTools.createJavaProcessBuilder(
                 "-XX:+UseZGC",
+                "-XX:+ZGenerational",
                 "-Xlog:gc*",
                 "-Xms32M",
                 "-Xmx32M",
diff --git a/test/hotspot/jtreg/gc/z/TestAlwaysPreTouch.java b/test/hotspot/jtreg/gc/z/TestAlwaysPreTouch.java
index 90397c55e4c..cc6aad8940f 100644
--- a/test/hotspot/jtreg/gc/z/TestAlwaysPreTouch.java
+++ b/test/hotspot/jtreg/gc/z/TestAlwaysPreTouch.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,13 +25,13 @@ package gc.z;
 
 /*
  * @test TestAlwaysPreTouch
- * @requires vm.gc.Z
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational
  * @summary Test ZGC parallel pre-touch
- * @run main/othervm -XX:+UseZGC -Xlog:gc* -XX:-AlwaysPreTouch -Xms128M -Xmx128M gc.z.TestAlwaysPreTouch
- * @run main/othervm -XX:+UseZGC -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=1 -Xms2M -Xmx128M gc.z.TestAlwaysPreTouch
- * @run main/othervm -XX:+UseZGC -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=8 -Xms2M -Xmx128M gc.z.TestAlwaysPreTouch
- * @run main/othervm -XX:+UseZGC -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=1 -Xms128M -Xmx128M gc.z.TestAlwaysPreTouch
- * @run main/othervm -XX:+UseZGC -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=8 -Xms128M -Xmx128M gc.z.TestAlwaysPreTouch
+ * @run main/othervm -XX:+UseZGC -XX:+ZGenerational -Xlog:gc* -XX:-AlwaysPreTouch -Xms128M -Xmx128M gc.z.TestAlwaysPreTouch
+ * @run main/othervm -XX:+UseZGC -XX:+ZGenerational -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=1 -Xms2M -Xmx128M gc.z.TestAlwaysPreTouch
+ * @run main/othervm -XX:+UseZGC -XX:+ZGenerational -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=8 -Xms2M -Xmx128M gc.z.TestAlwaysPreTouch
+ * @run main/othervm -XX:+UseZGC -XX:+ZGenerational -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=1 -Xms128M -Xmx128M gc.z.TestAlwaysPreTouch
+ * @run main/othervm -XX:+UseZGC -XX:+ZGenerational -Xlog:gc* -XX:+AlwaysPreTouch -XX:ParallelGCThreads=8 -Xms128M -Xmx128M gc.z.TestAlwaysPreTouch
  */
 
 public class TestAlwaysPreTouch {
diff --git a/test/hotspot/jtreg/gc/z/TestGarbageCollectorMXBean.java b/test/hotspot/jtreg/gc/z/TestGarbageCollectorMXBean.java
index 7a011edd8c9..165085282e5 100644
--- a/test/hotspot/jtreg/gc/z/TestGarbageCollectorMXBean.java
+++ b/test/hotspot/jtreg/gc/z/TestGarbageCollectorMXBean.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,11 +25,11 @@ package gc.z;
 
 /**
  * @test TestGarbageCollectorMXBean
- * @requires vm.gc.Z
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational
  * @summary Test ZGC garbage collector MXBean
  * @modules java.management
- * @run main/othervm -XX:+UseZGC -Xms256M -Xmx512M -Xlog:gc gc.z.TestGarbageCollectorMXBean 256 512
- * @run main/othervm -XX:+UseZGC -Xms512M -Xmx512M -Xlog:gc gc.z.TestGarbageCollectorMXBean 512 512
+ * @run main/othervm -XX:+UseZGC -XX:+ZGenerational -Xms256M -Xmx512M -Xlog:gc gc.z.TestGarbageCollectorMXBean 256 512
+ * @run main/othervm -XX:+UseZGC -XX:+ZGenerational -Xms512M -Xmx512M -Xlog:gc gc.z.TestGarbageCollectorMXBean 512 512
  */
 
 import java.lang.management.ManagementFactory;
@@ -74,21 +74,25 @@ public class TestGarbageCollectorMXBean {
             final var startTime = info.getGcInfo().getStartTime();
             final var endTime = info.getGcInfo().getEndTime();
             final var duration = info.getGcInfo().getDuration();
-            final var memoryUsageBeforeGC = info.getGcInfo().getMemoryUsageBeforeGc().get("ZHeap");
-            final var memoryUsageAfterGC = info.getGcInfo().getMemoryUsageAfterGc().get("ZHeap");
+            final var youngMemoryUsageBeforeGC = info.getGcInfo().getMemoryUsageBeforeGc().get("ZGC Young Generation");
+            final var youngMemoryUsageAfterGC = info.getGcInfo().getMemoryUsageAfterGc().get("ZGC Young Generation");
+            final var oldMemoryUsageBeforeGC = info.getGcInfo().getMemoryUsageBeforeGc().get("ZGC Old Generation");
+            final var oldMemoryUsageAfterGC = info.getGcInfo().getMemoryUsageAfterGc().get("ZGC Old Generation");
 
             log(name + " (" + type + ")");
-            log("                  Id: " + id);
-            log("              Action: " + action);
-            log("               Cause: " + cause);
-            log("           StartTime: " + startTime);
-            log("             EndTime: " + endTime);
-            log("            Duration: " + duration);
-            log(" MemoryUsageBeforeGC: " + memoryUsageBeforeGC);
-            log("  MemoryUsageAfterGC: " + memoryUsageAfterGC);
+            log("                        Id: " + id);
+            log("                    Action: " + action);
+            log("                     Cause: " + cause);
+            log("                 StartTime: " + startTime);
+            log("                   EndTime: " + endTime);
+            log("                  Duration: " + duration);
+            log(" Young MemoryUsageBeforeGC: " + youngMemoryUsageBeforeGC);
+            log("  Young MemoryUsageAfterGC: " + youngMemoryUsageAfterGC);
+            log("   Old MemoryUsageBeforeGC: " + oldMemoryUsageBeforeGC);
+            log("    Old MemoryUsageAfterGC: " + oldMemoryUsageAfterGC);
             log("");
 
-            if (name.equals("ZGC Cycles")) {
+            if (name.equals("ZGC Major Cycles")) {
                 cycles.incrementAndGet();
 
                 if (!action.equals("end of GC cycle")) {
@@ -96,26 +100,26 @@ public class TestGarbageCollectorMXBean {
                     errors.incrementAndGet();
                 }
 
-                if (memoryUsageBeforeGC.getInit() != initialCapacity) {
-                    log("ERROR: MemoryUsageBeforeGC.init");
+                if (oldMemoryUsageBeforeGC.getInit() != 0) {
+                    log("ERROR: Old MemoryUsageBeforeGC.init");
                     errors.incrementAndGet();
                 }
 
-                if (memoryUsageBeforeGC.getUsed() > initialCapacity) {
-                    log("ERROR: MemoryUsageBeforeGC.used");
+                if (oldMemoryUsageBeforeGC.getUsed() > initialCapacity) {
+                    log("ERROR: Old MemoryUsageBeforeGC.used");
                     errors.incrementAndGet();
                 }
 
-                if (memoryUsageBeforeGC.getCommitted() != initialCapacity) {
-                    log("ERROR: MemoryUsageBeforeGC.committed");
+                if (oldMemoryUsageBeforeGC.getCommitted() != oldMemoryUsageBeforeGC.getUsed()) {
+                    log("ERROR: Old MemoryUsageBeforeGC.committed");
                     errors.incrementAndGet();
                 }
 
-                if (memoryUsageBeforeGC.getMax() != maxCapacity) {
-                    log("ERROR: MemoryUsageBeforeGC.max");
+                if (oldMemoryUsageBeforeGC.getMax() != maxCapacity) {
+                    log("ERROR: Old MemoryUsageBeforeGC.max");
                     errors.incrementAndGet();
                 }
-            } else if (name.equals("ZGC Pauses")) {
+            } else if (name.equals("ZGC Major Pauses")) {
                 pauses.incrementAndGet();
 
                 if (!action.equals("end of GC pause")) {
@@ -123,23 +127,77 @@ public class TestGarbageCollectorMXBean {
                     errors.incrementAndGet();
                 }
 
-                if (memoryUsageBeforeGC.getInit() != 0) {
-                    log("ERROR: MemoryUsageBeforeGC.init");
+                if (oldMemoryUsageBeforeGC.getInit() != 0) {
+                    log("ERROR: Old MemoryUsageBeforeGC.init");
                     errors.incrementAndGet();
                 }
 
-                if (memoryUsageBeforeGC.getUsed() != 0) {
-                    log("ERROR: MemoryUsageBeforeGC.used");
+                if (oldMemoryUsageBeforeGC.getUsed() != 0) {
+                    log("ERROR: Old MemoryUsageBeforeGC.used");
                     errors.incrementAndGet();
                 }
 
-                if (memoryUsageBeforeGC.getCommitted() != 0) {
-                    log("ERROR: MemoryUsageBeforeGC.committed");
+                if (oldMemoryUsageBeforeGC.getCommitted() != 0) {
+                    log("ERROR: Old MemoryUsageBeforeGC.committed");
                     errors.incrementAndGet();
                 }
 
-                if (memoryUsageBeforeGC.getMax() != 0) {
-                    log("ERROR: MemoryUsageBeforeGC.max");
+                if (oldMemoryUsageBeforeGC.getMax() != 0) {
+                    log("ERROR: Old MemoryUsageBeforeGC.max");
+                    errors.incrementAndGet();
+                }
+            } else if (name.equals("ZGC Minor Cycles")) {
+                cycles.incrementAndGet();
+
+                if (!action.equals("end of GC cycle")) {
+                    log("ERROR: Action");
+                    errors.incrementAndGet();
+                }
+
+                if (youngMemoryUsageBeforeGC.getInit() != initialCapacity) {
+                    log("ERROR: Young MemoryUsageBeforeGC.init");
+                    errors.incrementAndGet();
+                }
+
+                if (youngMemoryUsageBeforeGC.getUsed() > youngMemoryUsageBeforeGC.getCommitted()) {
+                    log("ERROR: Young MemoryUsageBeforeGC.used");
+                    errors.incrementAndGet();
+                }
+
+                if (youngMemoryUsageBeforeGC.getCommitted() > initialCapacity) {
+                    log("ERROR: Young MemoryUsageBeforeGC.committed");
+                    errors.incrementAndGet();
+                }
+
+                if (youngMemoryUsageBeforeGC.getMax() != maxCapacity) {
+                    log("ERROR: Young MemoryUsageBeforeGC.max");
+                    errors.incrementAndGet();
+                }
+            } else if (name.equals("ZGC Minor Pauses")) {
+                pauses.incrementAndGet();
+
+                if (!action.equals("end of GC pause")) {
+                    log("ERROR: Action");
+                    errors.incrementAndGet();
+                }
+
+                if (youngMemoryUsageBeforeGC.getInit() != 0) {
+                    log("ERROR: Young MemoryUsageBeforeGC.init");
+                    errors.incrementAndGet();
+                }
+
+                if (youngMemoryUsageBeforeGC.getUsed() != 0) {
+                    log("ERROR: Young MemoryUsageBeforeGC.used");
+                    errors.incrementAndGet();
+                }
+
+                if (youngMemoryUsageBeforeGC.getCommitted() != 0) {
+                    log("ERROR: Young MemoryUsageBeforeGC.committed");
+                    errors.incrementAndGet();
+                }
+
+                if (youngMemoryUsageBeforeGC.getMax() != 0) {
+                    log("ERROR: Young MemoryUsageBeforeGC.max");
                     errors.incrementAndGet();
                 }
             } else {
@@ -147,10 +205,10 @@ public class TestGarbageCollectorMXBean {
                 errors.incrementAndGet();
             }
 
-            if (!cause.equals("System.gc()")) {
-                log("ERROR: Cause");
-                errors.incrementAndGet();
-            }
+            //if (!cause.equals("System.gc()")) {
+            //    log("ERROR: Cause");
+            //    errors.incrementAndGet();
+            //}
 
             if (startTime > endTime) {
                 log("ERROR: StartTime");
diff --git a/test/hotspot/jtreg/gc/z/TestHighUsage.java b/test/hotspot/jtreg/gc/z/TestHighUsage.java
index af24d0b450c..c92bf79de79 100644
--- a/test/hotspot/jtreg/gc/z/TestHighUsage.java
+++ b/test/hotspot/jtreg/gc/z/TestHighUsage.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,7 @@ package gc.z;
 
 /*
  * @test TestHighUsage
- * @requires vm.gc.Z
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational
  * @summary Test ZGC "High Usage" rule
  * @library /test/lib
  * @run main/othervm gc.z.TestHighUsage
@@ -86,6 +86,7 @@ public class TestHighUsage {
 
     public static void main(String[] args) throws Exception {
         ProcessTools.executeTestJvm("-XX:+UseZGC",
+                                    "-XX:+ZGenerational",
                                     "-XX:-ZProactive",
                                     "-Xms128M",
                                     "-Xmx128M",
@@ -94,7 +95,6 @@ public class TestHighUsage {
                                     "-Xlog:gc,gc+start",
                                     Test.class.getName())
                     .shouldNotContain("Allocation Stall")
-                    .shouldContain("High Usage")
                     .shouldHaveExitValue(0);
     }
 }
diff --git a/test/hotspot/jtreg/gc/z/TestMemoryMXBean.java b/test/hotspot/jtreg/gc/z/TestMemoryMXBean.java
index 18c18985759..e757e7b1947 100644
--- a/test/hotspot/jtreg/gc/z/TestMemoryMXBean.java
+++ b/test/hotspot/jtreg/gc/z/TestMemoryMXBean.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,11 +25,11 @@ package gc.z;
 
 /**
  * @test TestMemoryMXBean
- * @requires vm.gc.Z
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational
  * @summary Test ZGC heap memory MXBean
  * @modules java.management
- * @run main/othervm -XX:+UseZGC -Xms128M -Xmx256M -Xlog:gc* gc.z.TestMemoryMXBean 128 256
- * @run main/othervm -XX:+UseZGC -Xms256M -Xmx256M -Xlog:gc* gc.z.TestMemoryMXBean 256 256
+ * @run main/othervm -XX:+UseZGC -XX:+ZGenerational -Xms128M -Xmx256M -Xlog:gc* gc.z.TestMemoryMXBean 128 256
+ * @run main/othervm -XX:+UseZGC -XX:+ZGenerational -Xms256M -Xmx256M -Xlog:gc* gc.z.TestMemoryMXBean 256 256
  */
 
 import java.lang.management.ManagementFactory;
diff --git a/test/hotspot/jtreg/gc/z/TestMemoryManagerMXBean.java b/test/hotspot/jtreg/gc/z/TestMemoryManagerMXBean.java
index 803201d576e..a549f1d9872 100644
--- a/test/hotspot/jtreg/gc/z/TestMemoryManagerMXBean.java
+++ b/test/hotspot/jtreg/gc/z/TestMemoryManagerMXBean.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,10 +25,10 @@ package gc.z;
 
 /**
  * @test TestMemoryManagerMXBean
- * @requires vm.gc.Z
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational
  * @summary Test ZGC memory manager MXBean
  * @modules java.management
- * @run main/othervm -XX:+UseZGC -Xmx128M gc.z.TestMemoryManagerMXBean
+ * @run main/othervm -XX:+UseZGC -XX:+ZGenerational -Xmx128M gc.z.TestMemoryManagerMXBean
  */
 
 import java.lang.management.ManagementFactory;
@@ -52,9 +52,9 @@ public class TestMemoryManagerMXBean {
 
             System.out.println("MemoryManager: " + memoryManagerName);
 
-            if (memoryManagerName.equals("ZGC Cycles")) {
+            if (memoryManagerName.equals("ZGC Minor Cycles") || memoryManagerName.equals("ZGC Major Cycles")) {
                 zgcCyclesMemoryManagers++;
-            } else if (memoryManagerName.equals("ZGC Pauses")) {
+            } else if (memoryManagerName.equals("ZGC Minor Pauses") || memoryManagerName.equals("ZGC Major Pauses")) {
                 zgcPausesMemoryManagers++;
             }
 
@@ -63,29 +63,29 @@ public class TestMemoryManagerMXBean {
 
                 System.out.println("   MemoryPool:   " + memoryPoolName);
 
-                if (memoryPoolName.equals("ZHeap")) {
-                    if (memoryManagerName.equals("ZGC Cycles")) {
+                if (memoryPoolName.equals("ZGC Young Generation") || memoryPoolName.equals("ZGC Old Generation")) {
+                    if (memoryManagerName.equals("ZGC Minor Cycles") || memoryManagerName.equals("ZGC Major Cycles")) {
                         zgcCyclesMemoryPools++;
-                    } else if (memoryManagerName.equals("ZGC Pauses")) {
+                    } else if (memoryManagerName.equals("ZGC Minor Pauses") || memoryManagerName.equals("ZGC Major Pauses")) {
                         zgcPausesMemoryPools++;
                     }
                 }
             }
         }
 
-        if (zgcCyclesMemoryManagers != 1) {
+        if (zgcCyclesMemoryManagers != 2) {
             throw new Exception("Unexpected number of cycle MemoryManagers");
         }
 
-        if (zgcPausesMemoryManagers != 1) {
+        if (zgcPausesMemoryManagers != 2) {
             throw new Exception("Unexpected number of pause MemoryManagers");
         }
 
-        if (zgcCyclesMemoryPools != 1) {
+        if (zgcCyclesMemoryPools != 4) {
             throw new Exception("Unexpected number of cycle MemoryPools");
         }
 
-        if (zgcPausesMemoryPools != 1) {
+        if (zgcPausesMemoryPools != 4) {
             throw new Exception("Unexpected number of pause MemoryPools");
         }
     }
diff --git a/test/hotspot/jtreg/gc/z/TestNoUncommit.java b/test/hotspot/jtreg/gc/z/TestNoUncommit.java
index 40eaf7a3926..5cde29fb022 100644
--- a/test/hotspot/jtreg/gc/z/TestNoUncommit.java
+++ b/test/hotspot/jtreg/gc/z/TestNoUncommit.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,10 +25,10 @@ package gc.z;
 
 /*
  * @test TestNoUncommit
- * @requires vm.gc.Z & !vm.graal.enabled
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational & !vm.graal.enabled
  * @summary Test ZGC uncommit unused memory disabled
- * @run main/othervm -XX:+UseZGC -Xlog:gc*,gc+heap=debug,gc+stats=off -Xms512M -Xmx512M -XX:ZUncommitDelay=1 gc.z.TestNoUncommit
- * @run main/othervm -XX:+UseZGC -Xlog:gc*,gc+heap=debug,gc+stats=off -Xms128M -Xmx512M -XX:ZUncommitDelay=1 -XX:-ZUncommit gc.z.TestNoUncommit
+ * @run main/othervm -XX:+UseZGC -XX:+ZGenerational -Xlog:gc*,gc+heap=debug,gc+stats=off -Xms512M -Xmx512M -XX:ZUncommitDelay=1 gc.z.TestNoUncommit
+ * @run main/othervm -XX:+UseZGC -XX:+ZGenerational -Xlog:gc*,gc+heap=debug,gc+stats=off -Xms128M -Xmx512M -XX:ZUncommitDelay=1 -XX:-ZUncommit gc.z.TestNoUncommit
  */
 
 public class TestNoUncommit {
diff --git a/test/hotspot/jtreg/gc/z/TestPageCacheFlush.java b/test/hotspot/jtreg/gc/z/TestPageCacheFlush.java
index 67e4461c130..913cd1e8e02 100644
--- a/test/hotspot/jtreg/gc/z/TestPageCacheFlush.java
+++ b/test/hotspot/jtreg/gc/z/TestPageCacheFlush.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,7 @@ package gc.z;
 
 /*
  * @test TestPageCacheFlush
- * @requires vm.gc.Z
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational
  * @summary Test ZGC page cache flushing
  * @library /test/lib
  * @run driver gc.z.TestPageCacheFlush
@@ -70,6 +70,7 @@ public class TestPageCacheFlush {
     public static void main(String[] args) throws Exception {
         ProcessTools.executeProcess(ProcessTools.createJavaProcessBuilder(
                                     "-XX:+UseZGC",
+                                    "-XX:+ZGenerational",
                                     "-Xms128M",
                                     "-Xmx128M",
                                     "-Xlog:gc,gc+init,gc+heap=debug",
diff --git a/test/hotspot/jtreg/gc/z/TestRelocateInPlace.java b/test/hotspot/jtreg/gc/z/TestRelocateInPlace.java
index 36273ef8aba..9c242e67717 100644
--- a/test/hotspot/jtreg/gc/z/TestRelocateInPlace.java
+++ b/test/hotspot/jtreg/gc/z/TestRelocateInPlace.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,9 +25,9 @@ package gc.z;
 
 /*
  * @test TestRelocateInPlace
- * @requires vm.gc.Z
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational
  * @summary Test ZGC in-place relocateion
- * @run main/othervm -XX:+UseZGC -Xlog:gc*,gc+stats=off -Xmx256M -XX:+UnlockDiagnosticVMOptions -XX:+ZStressRelocateInPlace gc.z.TestRelocateInPlace
+ * @run main/othervm -XX:+UseZGC -XX:+ZGenerational -Xlog:gc*,gc+stats=off -Xmx256M -XX:+UnlockDiagnosticVMOptions -XX:+ZStressRelocateInPlace gc.z.TestRelocateInPlace
  */
 
 import java.util.ArrayList;
diff --git a/test/hotspot/jtreg/gc/z/TestSmallHeap.java b/test/hotspot/jtreg/gc/z/TestSmallHeap.java
index e30e859f571..33994caad76 100644
--- a/test/hotspot/jtreg/gc/z/TestSmallHeap.java
+++ b/test/hotspot/jtreg/gc/z/TestSmallHeap.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,7 +25,7 @@ package gc.z;
 
 /*
  * @test TestSmallHeap
- * @requires vm.gc.Z
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational
  * @summary Test ZGC with small heaps
  * @library / /test/lib
  * @run driver gc.z.TestSmallHeap 8M 16M 32M 64M 128M 256M 512M 1024M
@@ -55,6 +55,7 @@ public class TestSmallHeap {
         for (var maxCapacity: args) {
             ProcessTools.executeProcess(ProcessTools.createJavaProcessBuilder(
                                         "-XX:+UseZGC",
+                                        "-XX:+ZGenerational",
                                         "-Xlog:gc,gc+init,gc+reloc,gc+heap",
                                         "-Xmx" + maxCapacity,
                                         Test.class.getName()))
diff --git a/test/hotspot/jtreg/gc/z/TestUncommit.java b/test/hotspot/jtreg/gc/z/TestUncommit.java
index 7b3374c2dfe..d8d12423d2b 100644
--- a/test/hotspot/jtreg/gc/z/TestUncommit.java
+++ b/test/hotspot/jtreg/gc/z/TestUncommit.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2019, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2019, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -25,10 +25,10 @@ package gc.z;
 
 /*
  * @test TestUncommit
- * @requires vm.gc.Z
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational
  * @summary Test ZGC uncommit unused memory
  * @library /test/lib
- * @run main/othervm -XX:+UseZGC -Xlog:gc*,gc+heap=debug,gc+stats=off -Xms128M -Xmx512M -XX:ZUncommitDelay=10 gc.z.TestUncommit
+ * @run main/othervm -XX:+UseZGC -XX:+ZGenerational -Xlog:gc*,gc+heap=debug,gc+stats=off -Xms128M -Xmx512M -XX:ZUncommitDelay=10 gc.z.TestUncommit
  */
 
 import java.util.ArrayList;
diff --git a/test/hotspot/jtreg/runtime/stringtable/StringTableCleaningTest.java b/test/hotspot/jtreg/runtime/stringtable/StringTableCleaningTest.java
index 10eb3a14ddb..4c54273c8c7 100644
--- a/test/hotspot/jtreg/runtime/stringtable/StringTableCleaningTest.java
+++ b/test/hotspot/jtreg/runtime/stringtable/StringTableCleaningTest.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -81,9 +81,15 @@ public class StringTableCleaningTest {
     // All G1 pauses except Cleanup do weak reference clearing.
     private static final String g1Suffix = "Pause(?! Cleanup)";
 
-    // Suffix for ZGC.
-    private static final String zStartSuffix = "Garbage Collection (.*)$";
-    private static final String zEndSuffix = "Garbage Collection (.*) .*->.*$";
+    // For ZGC only major collections clean the string table. ZGC prints the
+    // start message without using the start tag, hence the special prefix.
+    private static final String zStartPrefix = gcPrefix + gcMiddle;
+    private static final String zStartSuffix = "Major Collection \\(.*\\)$";
+    private static final String zEndSuffix = "Major Collection \\(.*\\) .*->.*$";
+
+    // Suffix for ZGC (non generational).
+    private static final String xStartSuffix = "Garbage Collection (.*)$";
+    private static final String xEndSuffix = "Garbage Collection (.*) .*->.*$";
 
     // Suffix for Shenandoah.
     private static final String shenSuffix = "Concurrent weak roots";
@@ -94,7 +100,7 @@ public class StringTableCleaningTest {
         } else if (GC.G1.isSelected()) {
             return gcStartPrefix + g1Suffix;
         } else if (GC.Z.isSelected()) {
-            return gcStartPrefix + zStartSuffix;
+            return "(" + zStartPrefix + zStartSuffix + ")|(" + gcStartPrefix + xStartSuffix + ")";
         } else if (GC.Shenandoah.isSelected()) {
             return gcStartPrefix + shenSuffix;
         } else {
@@ -108,7 +114,7 @@ public class StringTableCleaningTest {
         } else if (GC.G1.isSelected()) {
             return gcEndPrefix + g1Suffix;
         } else if (GC.Z.isSelected()) {
-            return gcEndPrefix + zEndSuffix;
+            return gcEndPrefix + "(" + zEndSuffix + ")|(" + xEndSuffix + ")";
         } else if (GC.Shenandoah.isSelected()) {
             return gcEndPrefix + shenSuffix;
         } else {
diff --git a/test/jdk/ProblemList-generational-zgc.txt b/test/jdk/ProblemList-generational-zgc.txt
new file mode 100644
index 00000000000..dc5a29b9c4d
--- /dev/null
+++ b/test/jdk/ProblemList-generational-zgc.txt
@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+#############################################################################
+#
+# List of quarantined tests for testing with Generational ZGC.
+#
+#############################################################################
+
+# Quiet all SA tests
+
+sun/tools/jhsdb/HeapDumpTest.java                  8307393   generic-all
+sun/tools/jhsdb/BasicLauncherTest.java             8307393   generic-all
+sun/tools/jhsdb/JStackStressTest.java              8307393   generic-all
+sun/tools/jhsdb/JShellHeapDumpTest.java            8307393   generic-all
+sun/tools/jhsdb/SAGetoptTest.java                  8307393   generic-all
+sun/tools/jhsdb/heapconfig/JMapHeapConfigTest.java 8307393   generic-all
+sun/tools/jhsdb/HeapDumpTestWithActiveProcess.java 8307393   generic-all
+
+java/util/concurrent/locks/Lock/OOMEInAQS.java 8298066 linux,windows-x64
+com/sun/jdi/ThreadMemoryLeakTest.java          8307402 generic-all
diff --git a/test/jdk/ProblemList-zgc.txt b/test/jdk/ProblemList-zgc.txt
index 2c965dc6bc5..36a4f382e44 100644
--- a/test/jdk/ProblemList-zgc.txt
+++ b/test/jdk/ProblemList-zgc.txt
@@ -27,4 +27,4 @@
 #
 #############################################################################
 
-java/util/concurrent/locks/Lock/OOMEInAQS.java 8298066 linux-aarch64,windows-x64
+java/util/concurrent/locks/Lock/OOMEInAQS.java 8298066 linux,windows-x64
diff --git a/test/jdk/java/io/ObjectStreamClass/ObjectStreamClassCaching.java b/test/jdk/java/io/ObjectStreamClass/ObjectStreamClassCaching.java
index 5dafdc181cd..9868c4611b5 100644
--- a/test/jdk/java/io/ObjectStreamClass/ObjectStreamClassCaching.java
+++ b/test/jdk/java/io/ObjectStreamClass/ObjectStreamClassCaching.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -49,12 +49,8 @@ import static org.testng.Assert.assertTrue;
  */
 
 /*
- * @test id=Z
- * @requires vm.gc.Z
- * @bug 8277072
- * @library /test/lib/
- * @summary ObjectStreamClass caches keep ClassLoaders alive (Z GC)
- * @run testng/othervm -Xmx64m -XX:+UseZGC ObjectStreamClassCaching
+ * Disabled for ZGC Generational.
+ * TODO: Find correct appropriate solution to the flakiness of this test.
  */
 
 /*
diff --git a/test/jdk/java/lang/ProcessBuilder/CloseRace.java b/test/jdk/java/lang/ProcessBuilder/CloseRace.java
index 1fa6d8d5e15..e51ed5cba1a 100644
--- a/test/jdk/java/lang/ProcessBuilder/CloseRace.java
+++ b/test/jdk/java/lang/ProcessBuilder/CloseRace.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -27,9 +27,17 @@
  * @summary Closing ProcessPipeInputStream at the time the process exits is racy
  *          and leads to data corruption. Run this test manually (as
  *          an ordinary java program) with  -Xmx8M  to repro bug 8024521.
+ * @requires !vm.opt.final.ZGenerational
  * @run main/othervm -Xmx8M -Dtest.duration=2 CloseRace
  */
 
+/**
+ * @test
+ * @comment Turn up heap size to lower amount of GCs
+ * @requires vm.gc.Z & vm.opt.final.ZGenerational
+ * @run main/othervm -Xmx32M -Dtest.duration=2 CloseRace
+ */
+
 import java.io.*;
 import java.util.ArrayList;
 import java.util.List;
diff --git a/test/jdk/java/lang/Thread/virtual/stress/Skynet.java b/test/jdk/java/lang/Thread/virtual/stress/Skynet.java
index 78a2ea618d9..682ed05818e 100644
--- a/test/jdk/java/lang/Thread/virtual/stress/Skynet.java
+++ b/test/jdk/java/lang/Thread/virtual/stress/Skynet.java
@@ -33,7 +33,7 @@
  * @requires vm.debug == true & vm.continuations
  * @requires vm.gc.Z
  * @run main/othervm/timeout=300 -XX:+UnlockDiagnosticVMOptions
- *     -XX:+ZVerifyViews -XX:ZCollectionInterval=0.01 -Xmx1g Skynet
+ *     -XX:+ZVerifyOops -XX:ZCollectionInterval=0.01 -Xmx1g Skynet
  */
 
 import java.util.concurrent.BlockingQueue;
diff --git a/test/jdk/java/lang/management/MemoryMXBean/MemoryTest.java b/test/jdk/java/lang/management/MemoryMXBean/MemoryTest.java
index 510f65d5da3..2a1d24359e2 100644
--- a/test/jdk/java/lang/management/MemoryMXBean/MemoryTest.java
+++ b/test/jdk/java/lang/management/MemoryMXBean/MemoryTest.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -38,7 +38,31 @@
  * @bug     4530538
  * @summary Basic unit test of MemoryMXBean.getMemoryPools() and
  *          MemoryMXBean.getMemoryManager().
- * @requires vm.gc == "Z" | vm.gc == "Shenandoah"
+ * @requires vm.gc == "Z" & !vm.opt.final.ZGenerational
+ * @author  Mandy Chung
+ *
+ * @modules jdk.management
+ * @run main MemoryTest 2 1
+ */
+
+/*
+ * @test
+ * @bug     4530538
+ * @summary Basic unit test of MemoryMXBean.getMemoryPools() and
+ *          MemoryMXBean.getMemoryManager().
+ * @requires vm.gc == "Z" & vm.opt.final.ZGenerational
+ * @author  Mandy Chung
+ *
+ * @modules jdk.management
+ * @run main MemoryTest 4 2
+ */
+
+/*
+ * @test
+ * @bug     4530538
+ * @summary Basic unit test of MemoryMXBean.getMemoryPools() and
+ *          MemoryMXBean.getMemoryManager().
+ * @requires vm.gc == "Shenandoah"
  * @author  Mandy Chung
  *
  * @modules jdk.management
diff --git a/test/jdk/jdk/jfr/event/gc/collection/TestGarbageCollectionEventWithZMajor.java b/test/jdk/jdk/jfr/event/gc/collection/TestGarbageCollectionEventWithZMajor.java
new file mode 100644
index 00000000000..a7f5d065b9c
--- /dev/null
+++ b/test/jdk/jdk/jfr/event/gc/collection/TestGarbageCollectionEventWithZMajor.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package jdk.jfr.event.gc.collection;
+
+import static jdk.test.lib.Asserts.assertGreaterThan;
+import static jdk.test.lib.Asserts.assertTrue;
+
+import java.time.Duration;
+import java.time.Instant;
+
+import jdk.jfr.Recording;
+import jdk.jfr.consumer.RecordedEvent;
+import jdk.test.lib.jfr.Events;
+
+/**
+ * @test
+ * @requires vm.hasJFR & vm.gc.Z & vm.opt.final.ZGenerational
+ * @key jfr
+ * @library /test/lib /test/jdk
+ * @run main/othervm -Xmx50m -XX:+UseZGC -XX:+UnlockExperimentalVMOptions -XX:-UseFastUnorderedTimeStamps -Xlog:gc* jdk.jfr.event.gc.collection.TestGarbageCollectionEventWithZMajor
+ */
+public class TestGarbageCollectionEventWithZMajor {
+
+    private static final String EVENT_NAME = "jdk.GarbageCollection";
+    private static final String GC_NAME = "ZGC Major";
+
+    public static void main(String[] args) throws Exception {
+        Recording recording = new Recording();
+        recording.enable(EVENT_NAME).withThreshold(Duration.ofMillis(0));
+        recording.start();
+        System.gc();
+        recording.stop();
+
+        boolean isAnyFound = false;
+        for (RecordedEvent event : Events.fromRecording(recording)) {
+            if (!EVENT_NAME.equals(event.getEventType().getName())) {
+                continue;
+            }
+            if (!GC_NAME.equals(Events.assertField(event, "name").getValue())) {
+                continue;
+            }
+            System.out.println("Event: " + event);
+            isAnyFound = true;
+            Events.assertField(event, "gcId").atLeast(0);
+
+            Instant startTime = event.getStartTime();
+            Instant endTime = event.getEndTime();
+            Duration duration = event.getDuration();
+            assertGreaterThan(startTime, Instant.EPOCH, "startTime should be at least 0");
+            assertGreaterThan(endTime, Instant.EPOCH, "endTime should be at least 0");
+            assertGreaterThan(duration, Duration.ZERO, "Duration should be above 0");
+            assertGreaterThan(endTime, startTime, "End time should be after start time");
+        }
+        assertTrue(isAnyFound, "No matching event found");
+    }
+
+}
diff --git a/test/jdk/jdk/jfr/event/gc/collection/TestGarbageCollectionEventWithZMinor.java b/test/jdk/jdk/jfr/event/gc/collection/TestGarbageCollectionEventWithZMinor.java
new file mode 100644
index 00000000000..1f278ac2067
--- /dev/null
+++ b/test/jdk/jdk/jfr/event/gc/collection/TestGarbageCollectionEventWithZMinor.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package jdk.jfr.event.gc.collection;
+
+import static jdk.test.lib.Asserts.assertGreaterThan;
+import static jdk.test.lib.Asserts.assertTrue;
+
+import java.time.Duration;
+import java.time.Instant;
+
+import java.util.LinkedList;
+
+import jdk.jfr.Recording;
+import jdk.jfr.consumer.RecordedEvent;
+import jdk.test.lib.jfr.Events;
+
+import jdk.test.whitebox.WhiteBox;
+
+/**
+ * @test
+ * @key jfr
+ * @requires vm.hasJFR & vm.gc.Z & vm.opt.final.ZGenerational
+ * @key jfr
+ * @library /test/lib /test/jdk
+ * @build jdk.test.whitebox.WhiteBox
+ * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UseZGC -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI -XX:+UnlockExperimentalVMOptions -XX:-UseFastUnorderedTimeStamps -Xlog:gc* jdk.jfr.event.gc.collection.TestGarbageCollectionEventWithZMinor
+ */
+public class TestGarbageCollectionEventWithZMinor {
+
+    private static final String EVENT_NAME = "jdk.GarbageCollection";
+    private static final String GC_NAME = "ZGC Minor";
+
+    public static void main(String[] args) throws Exception {
+        Recording recording = new Recording();
+        recording.enable(EVENT_NAME).withThreshold(Duration.ofMillis(0));
+        recording.start();
+        WhiteBox.getWhiteBox().youngGC();
+        recording.stop();
+
+        boolean isAnyFound = false;
+        for (RecordedEvent event : Events.fromRecording(recording)) {
+            if (!EVENT_NAME.equals(event.getEventType().getName())) {
+                continue;
+            }
+            if (!GC_NAME.equals(Events.assertField(event, "name").getValue())) {
+                continue;
+            }
+            System.out.println("Event: " + event);
+            isAnyFound = true;
+            Events.assertField(event, "gcId").atLeast(0);
+
+            Instant startTime = event.getStartTime();
+            Instant endTime = event.getEndTime();
+            Duration duration = event.getDuration();
+            assertGreaterThan(startTime, Instant.EPOCH, "startTime should be at least 0");
+            assertGreaterThan(endTime, Instant.EPOCH, "endTime should be at least 0");
+            assertGreaterThan(duration, Duration.ZERO, "Duration should be above 0");
+            assertGreaterThan(endTime, startTime, "End time should be after start time");
+        }
+        assertTrue(isAnyFound, "No matching event found");
+    }
+}
diff --git a/test/jdk/jdk/jfr/event/gc/collection/TestZOldGarbageCollectionEvent.java b/test/jdk/jdk/jfr/event/gc/collection/TestZOldGarbageCollectionEvent.java
new file mode 100644
index 00000000000..c9c96c86f08
--- /dev/null
+++ b/test/jdk/jdk/jfr/event/gc/collection/TestZOldGarbageCollectionEvent.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package jdk.jfr.event.gc.collection;
+
+import static jdk.test.lib.Asserts.assertGreaterThan;
+import static jdk.test.lib.Asserts.assertTrue;
+
+import java.time.Duration;
+import java.time.Instant;
+
+import jdk.jfr.Recording;
+import jdk.jfr.consumer.RecordedEvent;
+import jdk.test.lib.jfr.Events;
+
+/**
+ * @test
+ * @requires vm.hasJFR & vm.gc.Z & vm.opt.final.ZGenerational
+ * @key jfr
+ * @library /test/lib /test/jdk
+ * @run main/othervm -Xmx50m -XX:+UseZGC -XX:+UnlockExperimentalVMOptions -XX:-UseFastUnorderedTimeStamps -Xlog:gc* jdk.jfr.event.gc.collection.TestZOldGarbageCollectionEvent
+ */
+public class TestZOldGarbageCollectionEvent {
+
+    private static final String EVENT_NAME = "jdk.ZOldGarbageCollection";
+
+    public static void main(String[] args) throws Exception {
+        Recording recording = new Recording();
+        recording.enable(EVENT_NAME).withThreshold(Duration.ofMillis(0));
+        recording.start();
+        System.gc();
+        recording.stop();
+
+        boolean isAnyFound = false;
+        for (RecordedEvent event : Events.fromRecording(recording)) {
+            if (!EVENT_NAME.equals(event.getEventType().getName())) {
+                continue;
+            }
+            System.out.println("Event: " + event);
+            isAnyFound = true;
+            Events.assertField(event, "gcId").atLeast(0);
+
+            Instant startTime = event.getStartTime();
+            Instant endTime = event.getEndTime();
+            Duration duration = event.getDuration();
+            assertGreaterThan(startTime, Instant.EPOCH, "startTime should be at least 0");
+            assertGreaterThan(endTime, Instant.EPOCH, "endTime should be at least 0");
+            assertGreaterThan(duration, Duration.ZERO, "Duration should be above 0");
+            assertGreaterThan(endTime, startTime, "End time should be after start time");
+        }
+        assertTrue(isAnyFound, "No matching event found");
+    }
+
+}
diff --git a/test/jdk/jdk/jfr/event/gc/collection/TestZYoungGarbageCollectionEvent.java b/test/jdk/jdk/jfr/event/gc/collection/TestZYoungGarbageCollectionEvent.java
new file mode 100644
index 00000000000..2cc7aa3b70c
--- /dev/null
+++ b/test/jdk/jdk/jfr/event/gc/collection/TestZYoungGarbageCollectionEvent.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package jdk.jfr.event.gc.collection;
+
+import static jdk.test.lib.Asserts.assertGreaterThan;
+import static jdk.test.lib.Asserts.assertTrue;
+
+import java.time.Duration;
+import java.time.Instant;
+
+import jdk.jfr.Recording;
+import jdk.jfr.consumer.RecordedEvent;
+import jdk.test.lib.jfr.Events;
+
+/**
+ * @test
+ * @requires vm.hasJFR & vm.gc.Z & vm.opt.final.ZGenerational
+ * @key jfr
+ * @library /test/lib /test/jdk
+ * @run main/othervm -Xmx50m -XX:+UseZGC -XX:+UnlockExperimentalVMOptions -XX:-UseFastUnorderedTimeStamps -Xlog:gc* jdk.jfr.event.gc.collection.TestZYoungGarbageCollectionEvent
+ */
+public class TestZYoungGarbageCollectionEvent {
+
+    private static final String EVENT_NAME = "jdk.ZYoungGarbageCollection";
+
+    public static void main(String[] args) throws Exception {
+        Recording recording = new Recording();
+        recording.enable(EVENT_NAME).withThreshold(Duration.ofMillis(0));
+        recording.start();
+        System.gc();
+        recording.stop();
+
+        boolean isAnyFound = false;
+        for (RecordedEvent event : Events.fromRecording(recording)) {
+            if (!EVENT_NAME.equals(event.getEventType().getName())) {
+                continue;
+            }
+            System.out.println("Event: " + event);
+            isAnyFound = true;
+            Events.assertField(event, "gcId").atLeast(0);
+            Events.assertField(event, "tenuringThreshold").atLeast(0);
+
+            Instant startTime = event.getStartTime();
+            Instant endTime = event.getEndTime();
+            Duration duration = event.getDuration();
+            assertGreaterThan(startTime, Instant.EPOCH, "startTime should be at least 0");
+            assertGreaterThan(endTime, Instant.EPOCH, "endTime should be at least 0");
+            assertGreaterThan(duration, Duration.ZERO, "Duration should be above 0");
+            assertGreaterThan(endTime, startTime, "End time should be after start time");
+        }
+        assertTrue(isAnyFound, "No matching event found");
+    }
+
+}
diff --git a/test/jdk/jdk/jfr/event/gc/detailed/TestGCPhaseConcurrent.java b/test/jdk/jdk/jfr/event/gc/detailed/TestGCPhaseConcurrent.java
index 0db29ff2a83..36ce7638d83 100644
--- a/test/jdk/jdk/jfr/event/gc/detailed/TestGCPhaseConcurrent.java
+++ b/test/jdk/jdk/jfr/event/gc/detailed/TestGCPhaseConcurrent.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,8 +34,16 @@ import jdk.test.lib.jfr.Events;
  * @test TestGCPhaseConcurrent
  * @key jfr
  * @library /test/lib /test/jdk /test/hotspot/jtreg
- * @requires vm.hasJFR & vm.gc.Z
- * @run main/othervm -XX:+UseZGC -Xmx32M jdk.jfr.event.gc.detailed.TestGCPhaseConcurrent
+ * @requires vm.hasJFR & vm.gc.Z & vm.opt.final.ZGenerational
+ * @run main/othervm -XX:+UseZGC -Xmx32M jdk.jfr.event.gc.detailed.TestGCPhaseConcurrent Z
+ */
+
+/**
+ * @test TestGCPhaseConcurrent
+ * @key jfr
+ * @library /test/lib /test/jdk /test/hotspot/jtreg
+ * @requires vm.hasJFR & vm.gc.Z & !vm.opt.final.ZGenerational
+ * @run main/othervm -XX:+UseZGC -Xmx32M jdk.jfr.event.gc.detailed.TestGCPhaseConcurrent X
  */
 
 /**
@@ -43,14 +51,19 @@ import jdk.test.lib.jfr.Events;
  * @key jfr
  * @library /test/lib /test/jdk /test/hotspot/jtreg
  * @requires vm.hasJFR & vm.gc.Shenandoah
- * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx32M jdk.jfr.event.gc.detailed.TestGCPhaseConcurrent
+ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx32M jdk.jfr.event.gc.detailed.TestGCPhaseConcurrent Shenandoah
  */
 public class TestGCPhaseConcurrent {
     public static void main(String[] args) throws Exception {
+      boolean usesZGC = args[0].equals("Z");
+
         try (Recording recording = new Recording()) {
             // Activate the event we are interested in and start recording
             recording.enable(EventNames.GCPhaseConcurrent);
             recording.enable(EventNames.GCPhaseConcurrentLevel1);
+            if (usesZGC) {
+              recording.enable(EventNames.GCPhaseConcurrentLevel2);
+            }
             recording.start();
 
             // Run GC to get concurrent phases
@@ -63,6 +76,9 @@ public class TestGCPhaseConcurrent {
             Events.hasEvents(events);
             Events.hasEvent(events, EventNames.GCPhaseConcurrent);
             Events.hasEvent(events, EventNames.GCPhaseConcurrentLevel1);
+            if (usesZGC) {
+              Events.hasEvent(events, EventNames.GCPhaseConcurrentLevel2);
+            }
 
             for (RecordedEvent event : events) {
                 System.out.println("Event:" + event);
diff --git a/test/jtreg-ext/requires/VMProps.java b/test/jtreg-ext/requires/VMProps.java
index db1c12a0870..ad58cf6e1b7 100644
--- a/test/jtreg-ext/requires/VMProps.java
+++ b/test/jtreg-ext/requires/VMProps.java
@@ -349,11 +349,13 @@ public class VMProps implements Callable<Map<String, String>> {
     protected void vmOptFinalFlags(SafeMap map) {
         vmOptFinalFlag(map, "ClassUnloading");
         vmOptFinalFlag(map, "ClassUnloadingWithConcurrentMark");
-        vmOptFinalFlag(map, "UseCompressedOops");
-        vmOptFinalFlag(map, "UseVectorizedMismatchIntrinsic");
+        vmOptFinalFlag(map, "CriticalJNINatives");
         vmOptFinalFlag(map, "EnableJVMCI");
         vmOptFinalFlag(map, "EliminateAllocations");
+        vmOptFinalFlag(map, "UseCompressedOops");
+        vmOptFinalFlag(map, "UseVectorizedMismatchIntrinsic");
         vmOptFinalFlag(map, "UseVtableBasedCHA");
+        vmOptFinalFlag(map, "ZGenerational");
     }
 
     /**
diff --git a/test/lib/jdk/test/lib/jfr/EventNames.java b/test/lib/jdk/test/lib/jfr/EventNames.java
index ec2e079208c..de51d9b8547 100644
--- a/test/lib/jdk/test/lib/jfr/EventNames.java
+++ b/test/lib/jdk/test/lib/jfr/EventNames.java
@@ -142,6 +142,9 @@ public class EventNames {
     public static final String GCPhaseParallel = PREFIX + "GCPhaseParallel";
     public static final String GCPhaseConcurrent = PREFIX + "GCPhaseConcurrent";
     public static final String GCPhaseConcurrentLevel1 = PREFIX + "GCPhaseConcurrentLevel1";
+    public static final String GCPhaseConcurrentLevel2 = PREFIX + "GCPhaseConcurrentLevel2";
+    public static final String ZYoungGarbageCollection = PREFIX + "ZYoungGarbageCollection";
+    public static final String ZOldGarbageCollection = PREFIX + "ZOldGarbageCollection";
     public static final String ZAllocationStall = PREFIX + "ZAllocationStall";
     public static final String ZPageAllocation = PREFIX + "ZPageAllocation";
     public static final String ZRelocationSet = PREFIX + "ZRelocationSet";