8315149: Add hsperf counters for CPU time of internal GC threads

Co-authored-by: Man Cao <manc@openjdk.org>
Co-authored-by: Stefan Johansson <sjohanss@openjdk.org>
Reviewed-by: simonis, manc, sjohanss

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -103,6 +103,7 @@
 #include "oops/compressedOops.inline.hpp"
 #include "oops/oop.inline.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/cpuTimeCounters.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/init.hpp"
 #include "runtime/java.hpp"
@@ -1433,6 +1434,11 @@ jint G1CollectedHeap::initialize() {
 
   evac_failure_injector()->reset();
 
+  CPUTimeCounters::create_counter(CPUTimeGroups::CPUTimeType::gc_parallel_workers);
+  CPUTimeCounters::create_counter(CPUTimeGroups::CPUTimeType::gc_conc_mark);
+  CPUTimeCounters::create_counter(CPUTimeGroups::CPUTimeType::gc_conc_refine);
+  CPUTimeCounters::create_counter(CPUTimeGroups::CPUTimeType::gc_service);
+
   G1InitLogger::print();
 
   return JNI_OK;
@@ -2224,6 +2230,8 @@ void G1CollectedHeap::gc_epilogue(bool full) {
 
   _free_arena_memory_task->notify_new_stats(&_young_gen_card_set_stats,
                                             &_collection_set_candidates_card_set_stats);
+
+  update_parallel_gc_threads_cpu_time();
 }
 
 uint G1CollectedHeap::uncommit_regions(uint region_limit) {
@@ -2308,6 +2316,26 @@ void G1CollectedHeap::verify_region_attr_remset_is_tracked() {
 }
 #endif
 
+void G1CollectedHeap::update_parallel_gc_threads_cpu_time() {
+  assert(Thread::current()->is_VM_thread(),
+         "Must be called from VM thread to avoid races");
+  if (!UsePerfData || !os::is_thread_cpu_time_supported()) {
+    return;
+  }
+
+  // Ensure ThreadTotalCPUTimeClosure destructor is called before publishing gc
+  // time.
+  {
+    ThreadTotalCPUTimeClosure tttc(CPUTimeGroups::CPUTimeType::gc_parallel_workers);
+    // Currently parallel worker threads never terminate (JDK-8081682), so it is
+    // safe for VMThread to read their CPU times. However, if JDK-8087340 is
+    // resolved so they terminate, we should rethink if it is still safe.
+    workers()->threads_do(&tttc);
+  }
+
+  CPUTimeCounters::publish_gc_total_cpu_time();
+}
+
 void G1CollectedHeap::start_new_collection_set() {
   collection_set()->start_incremental_building();
 

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -264,6 +264,7 @@ public:
   void set_collection_set_candidates_stats(G1MonotonicArenaMemoryStats& stats);
   void set_young_gen_card_set_stats(const G1MonotonicArenaMemoryStats& stats);
 
+  void update_parallel_gc_threads_cpu_time();
 private:
 
   G1HRPrinter _hr_printer;

src/hotspot/share/gc/g1/g1ConcurrentMark.cpp

@@ -1336,6 +1336,8 @@ void G1ConcurrentMark::remark() {
   _remark_weak_ref_times.add((now - mark_work_end) * 1000.0);
   _remark_times.add((now - start) * 1000.0);
 
+  _g1h->update_parallel_gc_threads_cpu_time();
+
   policy->record_concurrent_mark_remark_end();
 }
 

src/hotspot/share/gc/g1/g1ConcurrentMarkThread.cpp

@@ -39,6 +39,7 @@
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "logging/log.hpp"
 #include "memory/resourceArea.hpp"
+#include "runtime/cpuTimeCounters.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/vmThread.hpp"
 #include "utilities/debug.hpp"
@@ -134,6 +135,8 @@ void G1ConcurrentMarkThread::run_service() {
     concurrent_cycle_end(_state == FullMark && !_cm->has_aborted());
 
     _vtime_accum = (os::elapsedVTime() - _vtime_start);
+
+    update_threads_cpu_time();
   }
   _cm->root_regions()->cancel_scan();
 }
@@ -169,6 +172,7 @@ bool G1ConcurrentMarkThread::phase_clear_cld_claimed_marks() {
 bool G1ConcurrentMarkThread::phase_scan_root_regions() {
   G1ConcPhaseTimer p(_cm, "Concurrent Scan Root Regions");
   _cm->scan_root_regions();
+  update_threads_cpu_time();
   return _cm->has_aborted();
 }
 
@@ -228,6 +232,7 @@ bool G1ConcurrentMarkThread::subphase_delay_to_keep_mmu_before_remark() {
 
 bool G1ConcurrentMarkThread::subphase_remark() {
   ConcurrentGCBreakpoints::at("BEFORE MARKING COMPLETED");
+  update_threads_cpu_time();
   VM_G1PauseRemark op;
   VMThread::execute(&op);
   return _cm->has_aborted();
@@ -237,6 +242,7 @@ bool G1ConcurrentMarkThread::phase_rebuild_and_scrub() {
   ConcurrentGCBreakpoints::at("AFTER REBUILD STARTED");
   G1ConcPhaseTimer p(_cm, "Concurrent Rebuild Remembered Sets and Scrub Regions");
   _cm->rebuild_and_scrub();
+  update_threads_cpu_time();
   return _cm->has_aborted();
 }
 
@@ -336,3 +342,12 @@ void G1ConcurrentMarkThread::concurrent_cycle_end(bool mark_cycle_completed) {
   _cm->concurrent_cycle_end(mark_cycle_completed);
   ConcurrentGCBreakpoints::notify_active_to_idle();
 }
+
+void G1ConcurrentMarkThread::update_threads_cpu_time() {
+  if (!UsePerfData || !os::is_thread_cpu_time_supported()) {
+    return;
+  }
+  ThreadTotalCPUTimeClosure tttc(CPUTimeGroups::CPUTimeType::gc_conc_mark);
+  tttc.do_thread(this);
+  _cm->threads_do(&tttc);
+}

src/hotspot/share/gc/g1/g1ConcurrentMarkThread.hpp

@@ -108,6 +108,9 @@ class G1ConcurrentMarkThread: public ConcurrentGCThread {
   bool in_progress() const;
 
   bool in_undo_mark() const;
+
+  // Update the perf data counter for concurrent mark.
+  void update_threads_cpu_time();
 };
 
 #endif // SHARE_GC_G1_G1CONCURRENTMARKTHREAD_HPP

src/hotspot/share/gc/g1/g1ConcurrentRefineThread.cpp

@@ -30,6 +30,7 @@
 #include "gc/g1/g1DirtyCardQueue.hpp"
 #include "gc/shared/suspendibleThreadSet.hpp"
 #include "logging/log.hpp"
+#include "runtime/cpuTimeCounters.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/os.hpp"
 #include "runtime/thread.hpp"
@@ -74,11 +75,7 @@ void G1ConcurrentRefineThread::run_service() {
       }
     }
     report_inactive("Deactivated", _refinement_stats - active_stats_start);
-    if (os::supports_vtime()) {
-      _vtime_accum = (os::elapsedVTime() - _vtime_start);
-    } else {
-      _vtime_accum = 0.0;
-    }
+    track_usage();
   }
 
   log_debug(gc, refine)("Stopping %d", _worker_id);
@@ -137,6 +134,7 @@ class G1PrimaryConcurrentRefineThread final : public G1ConcurrentRefineThread {
   bool wait_for_completed_buffers() override;
   bool maybe_deactivate() override;
   void do_refinement_step() override;
+  void track_usage() override;
 
 public:
   G1PrimaryConcurrentRefineThread(G1ConcurrentRefine* cr) :
@@ -182,6 +180,15 @@ void G1PrimaryConcurrentRefineThread::do_refinement_step() {
   }
 }
 
+void G1PrimaryConcurrentRefineThread::track_usage() {
+  G1ConcurrentRefineThread::track_usage();
+  // The primary thread is responsible for updating the CPU time for all workers.
+  if (UsePerfData && os::is_thread_cpu_time_supported()) {
+    ThreadTotalCPUTimeClosure tttc(CPUTimeGroups::CPUTimeType::gc_conc_refine);
+    cr()->threads_do(&tttc);
+  }
+}
+
 class G1SecondaryConcurrentRefineThread final : public G1ConcurrentRefineThread {
   bool wait_for_completed_buffers() override;
   void do_refinement_step() override;

src/hotspot/share/gc/g1/g1ConcurrentRefineThread.hpp

@@ -71,6 +71,16 @@ protected:
   // precondition: this is the current thread.
   virtual void do_refinement_step() = 0;
 
+  // Update concurrent refine threads stats.
+  // If we are in Primary thread, we additionally update CPU time tracking.
+  virtual void track_usage() {
+    if (os::supports_vtime()) {
+      _vtime_accum = (os::elapsedVTime() - _vtime_start);
+    } else {
+      _vtime_accum = 0.0;
+    }
+  };
+
   // Helper for do_refinement_step implementations.  Try to perform some
   // refinement work, limited by stop_at.  Returns true if any refinement work
   // was performed, false if no work available per stop_at.

src/hotspot/share/gc/g1/g1ServiceThread.cpp

@@ -25,6 +25,7 @@
 #include "precompiled.hpp"
 #include "gc/g1/g1ServiceThread.hpp"
 #include "logging/log.hpp"
+#include "runtime/cpuTimeCounters.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "runtime/timer.hpp"
 #include "runtime/os.hpp"
@@ -130,6 +131,8 @@ void G1ServiceThread::run_task(G1ServiceTask* task) {
 
   task->execute();
 
+  update_thread_cpu_time();
+
   log_debug(gc, task)("G1 Service Thread (%s) (run: %1.3fms) (cpu: %1.3fms)",
                       task->name(),
                       TimeHelper::counter_to_millis(os::elapsed_counter() - start),
@@ -151,6 +154,13 @@ void G1ServiceThread::stop_service() {
   ml.notify();
 }
 
+void G1ServiceThread::update_thread_cpu_time() {
+  if (UsePerfData && os::is_thread_cpu_time_supported()) {
+    ThreadTotalCPUTimeClosure tttc(CPUTimeGroups::CPUTimeType::gc_service);
+    tttc.do_thread(this);
+  }
+}
+
 G1ServiceTask::G1ServiceTask(const char* name) :
   _time(),
   _name(name),

src/hotspot/share/gc/g1/g1ServiceThread.hpp

@@ -120,6 +120,9 @@ class G1ServiceThread: public ConcurrentGCThread {
   // to schedule a registered task to run after the given delay.
   void schedule(G1ServiceTask* task, jlong delay, bool notify);
 
+  // Update the perf data counter for service thread.
+  void update_thread_cpu_time();
+
 public:
   G1ServiceThread();
 

src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp

@@ -49,6 +49,7 @@
 #include "memory/universe.hpp"
 #include "nmt/memTracker.hpp"
 #include "oops/oop.inline.hpp"
+#include "runtime/cpuTimeCounters.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/java.hpp"
 #include "runtime/vmThread.hpp"
@@ -127,6 +128,9 @@ jint ParallelScavengeHeap::initialize() {
     return JNI_ENOMEM;
   }
 
+  // Create CPU time counter
+  CPUTimeCounters::create_counter(CPUTimeGroups::CPUTimeType::gc_parallel_workers);
+
   ParallelInitLogger::print();
 
   return JNI_OK;
@@ -192,6 +196,7 @@ void ParallelScavengeHeap::update_counters() {
   young_gen()->update_counters();
   old_gen()->update_counters();
   MetaspaceCounters::update_performance_counters();
+  update_parallel_worker_threads_cpu_time();
 }
 
 size_t ParallelScavengeHeap::capacity() const {
@@ -884,3 +889,23 @@ void ParallelScavengeHeap::pin_object(JavaThread* thread, oop obj) {
 void ParallelScavengeHeap::unpin_object(JavaThread* thread, oop obj) {
   GCLocker::unlock_critical(thread);
 }
+
+void ParallelScavengeHeap::update_parallel_worker_threads_cpu_time() {
+  assert(Thread::current()->is_VM_thread(),
+         "Must be called from VM thread to avoid races");
+  if (!UsePerfData || !os::is_thread_cpu_time_supported()) {
+    return;
+  }
+
+  // Ensure ThreadTotalCPUTimeClosure destructor is called before publishing gc
+  // time.
+  {
+    ThreadTotalCPUTimeClosure tttc(CPUTimeGroups::CPUTimeType::gc_parallel_workers);
+    // Currently parallel worker threads in GCTaskManager never terminate, so it
+    // is safe for VMThread to read their CPU times. If upstream changes this
+    // behavior, we should rethink if it is still safe.
+    gc_threads_do(&tttc);
+  }
+
+  CPUTimeCounters::publish_gc_total_cpu_time();
+}

src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp

@@ -101,6 +101,8 @@ class ParallelScavengeHeap : public CollectedHeap {
   // Allocate in oldgen and record the allocation with the size_policy.
   HeapWord* allocate_old_gen_and_record(size_t word_size);
 
+  void update_parallel_worker_threads_cpu_time();
+
  protected:
   HeapWord* allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) override;
 

src/hotspot/share/gc/shared/stringdedup/stringDedupProcessor.cpp

@@ -38,6 +38,7 @@
 #include "memory/iterator.hpp"
 #include "oops/access.inline.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/cpuTimeCounters.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/mutexLocker.hpp"
 #include "utilities/debug.hpp"
@@ -64,6 +65,7 @@ StringDedup::Processor::Processor() : _thread(nullptr) {}
 
 void StringDedup::Processor::initialize() {
   _processor = new Processor();
+  CPUTimeCounters::create_counter(CPUTimeGroups::CPUTimeType::conc_dedup);
 }
 
 void StringDedup::Processor::wait_for_requests() const {
@@ -187,6 +189,10 @@ void StringDedup::Processor::run(JavaThread* thread) {
     cleanup_table(false /* grow_only */, StringDeduplicationResizeALot /* force */);
     _cur_stat.report_active_end();
     log_statistics();
+    if (UsePerfData && os::is_thread_cpu_time_supported()) {
+      ThreadTotalCPUTimeClosure tttc(CPUTimeGroups::CPUTimeType::conc_dedup);
+      tttc.do_thread(thread);
+    }
   }
 }
 

src/hotspot/share/memory/universe.cpp

@@ -68,6 +68,7 @@
 #include "prims/resolvedMethodTable.hpp"
 #include "runtime/arguments.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/cpuTimeCounters.hpp"
 #include "runtime/flags/jvmFlagLimit.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/init.hpp"
@@ -781,6 +782,9 @@ jint universe_init() {
 
   GCLogPrecious::initialize();
 
+  // Initialize CPUTimeCounters object, which must be done before creation of the heap.
+  CPUTimeCounters::initialize();
+
 #ifdef _LP64
   MetaspaceShared::adjust_heap_sizes_for_dumping();
 #endif // _LP64

src/hotspot/share/runtime/cpuTimeCounters.cpp

@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023 Google LLC. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+#include "precompiled.hpp"
+#include "runtime/cpuTimeCounters.hpp"
+#include "runtime/atomic.hpp"
+
+const char* CPUTimeGroups::to_string(CPUTimeType val) {
+  switch (val) {
+    case CPUTimeType::gc_total:
+      return "total_gc_cpu_time";
+    case CPUTimeType::gc_parallel_workers:
+      return "gc_parallel_workers";
+    case CPUTimeType::gc_conc_mark:
+      return "gc_conc_mark";
+    case CPUTimeType::gc_conc_refine:
+      return "gc_conc_refine";
+    case CPUTimeType::gc_service:
+      return "gc_service";
+    case CPUTimeType::vm:
+      return "vm";
+    case CPUTimeType::conc_dedup:
+      return "conc_dedup";
+    default:
+      ShouldNotReachHere();
+      return "";
+  };
+}
+
+bool CPUTimeGroups::is_gc_counter(CPUTimeType val) {
+  switch (val) {
+    case CPUTimeType::gc_parallel_workers:
+    case CPUTimeType::gc_conc_mark:
+    case CPUTimeType::gc_conc_refine:
+    case CPUTimeType::gc_service:
+      return true;
+    default:
+      return false;
+  }
+  ShouldNotReachHere();
+}
+
+CPUTimeCounters* CPUTimeCounters::_instance = nullptr;
+
+CPUTimeCounters::CPUTimeCounters() :
+    _cpu_time_counters(),
+    _gc_total_cpu_time_diff(0) {
+}
+
+void CPUTimeCounters::inc_gc_total_cpu_time(jlong diff) {
+  CPUTimeCounters* instance = CPUTimeCounters::get_instance();
+  Atomic::add(&(instance->_gc_total_cpu_time_diff), diff);
+}
+
+void CPUTimeCounters::publish_gc_total_cpu_time() {
+  CPUTimeCounters* instance = CPUTimeCounters::get_instance();
+  // Ensure that we are only incrementing atomically by using Atomic::cmpxchg
+  // to set the value to zero after we obtain the new CPU time difference.
+  jlong old_value;
+  jlong fetched_value = Atomic::load(&(instance->_gc_total_cpu_time_diff));
+  jlong new_value = 0;
+  do {
+    old_value = fetched_value;
+    fetched_value = Atomic::cmpxchg(&(instance->_gc_total_cpu_time_diff), old_value, new_value);
+  } while (old_value != fetched_value);
+  get_counter(CPUTimeGroups::CPUTimeType::gc_total)->inc(fetched_value);
+}
+
+void CPUTimeCounters::create_counter(CounterNS ns, CPUTimeGroups::CPUTimeType name) {
+  if (UsePerfData && os::is_thread_cpu_time_supported()) {
+    EXCEPTION_MARK;
+    CPUTimeCounters* instance = CPUTimeCounters::get_instance();
+    instance->_cpu_time_counters[static_cast<int>(name)] =
+                PerfDataManager::create_counter(ns, CPUTimeGroups::to_string(name),
+                                                PerfData::U_Ticks, CHECK);
+  }
+}
+
+void CPUTimeCounters::create_counter(CPUTimeGroups::CPUTimeType group) {
+  CPUTimeCounters::create_counter(SUN_THREADS_CPUTIME, group);
+}
+
+PerfCounter* CPUTimeCounters::get_counter(CPUTimeGroups::CPUTimeType name) {
+  return CPUTimeCounters::get_instance()->_cpu_time_counters[static_cast<int>(name)];
+}
+
+void CPUTimeCounters::update_counter(CPUTimeGroups::CPUTimeType name, jlong total) {
+  CPUTimeCounters* instance = CPUTimeCounters::get_instance();
+  PerfCounter* counter = instance->get_counter(name);
+  jlong prev_value = counter->get_value();
+  jlong net_cpu_time = total - prev_value;
+  counter->inc(net_cpu_time);
+  if (CPUTimeGroups::is_gc_counter(name)) {
+    instance->inc_gc_total_cpu_time(net_cpu_time);
+  }
+}
+
+ThreadTotalCPUTimeClosure::~ThreadTotalCPUTimeClosure() {
+  CPUTimeCounters::update_counter(_name, _total);
+}
+
+void ThreadTotalCPUTimeClosure::do_thread(Thread* thread) {
+  // The default code path (fast_thread_cpu_time()) asserts that
+  // pthread_getcpuclockid() and clock_gettime() must return 0. Thus caller
+  // must ensure the thread exists and has not terminated.
+  _total += os::thread_cpu_time(thread);
+}
+
+

src/hotspot/share/runtime/cpuTimeCounters.hpp

@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023 Google LLC. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+
+#ifndef SHARE_RUNTIME_CPUTIMECOUNTERS_HPP
+#define SHARE_RUNTIME_CPUTIMECOUNTERS_HPP
+
+
+#include "memory/iterator.hpp"
+#include "runtime/os.hpp"
+#include "runtime/perfData.hpp"
+#include "runtime/perfDataTypes.hpp"
+
+class CPUTimeGroups : public AllStatic {
+public:
+  enum class CPUTimeType {
+    gc_total,
+    gc_parallel_workers,
+    gc_conc_mark,
+    gc_conc_refine,
+    gc_service,
+    vm,
+    conc_dedup,
+    COUNT,
+  };
+
+  static const char* to_string(CPUTimeType val);
+  static bool is_gc_counter(CPUTimeType val);
+};
+
+class CPUTimeCounters: public CHeapObj<mtServiceability> {
+private:
+  // CPUTimeCounters is a singleton instance.
+  CPUTimeCounters();
+  NONCOPYABLE(CPUTimeCounters);
+
+  static CPUTimeCounters* _instance;
+
+  // An array of PerfCounters which correspond to the various counters we want
+  // to track. Indexed by the enum value `CPUTimeType`.
+  PerfCounter* _cpu_time_counters[static_cast<int>(CPUTimeGroups::CPUTimeType::COUNT)];
+
+  // A long which atomically tracks how much CPU time has been spent doing GC
+  // since the last time we called `publish_total_cpu_time()`.
+  // It is incremented using Atomic::add() to prevent race conditions, and
+  // is added to the `gc_total` CPUTimeType at the end of GC.
+  volatile jlong _gc_total_cpu_time_diff;
+
+  static void create_counter(CounterNS ns, CPUTimeGroups::CPUTimeType name);
+
+  static CPUTimeCounters* get_instance() {
+    assert(_instance != nullptr, "no instance found");
+    return _instance;
+  }
+
+  static void inc_gc_total_cpu_time(jlong diff);
+
+public:
+  static void initialize() {
+    assert(_instance == nullptr, "we can only allocate one CPUTimeCounters object");
+    if (UsePerfData && os::is_thread_cpu_time_supported()) {
+      _instance = new CPUTimeCounters();
+      create_counter(SUN_THREADS, CPUTimeGroups::CPUTimeType::gc_total);
+    }
+  }
+
+  static void create_counter(CPUTimeGroups::CPUTimeType name);
+  static PerfCounter* get_counter(CPUTimeGroups::CPUTimeType name);
+  static void update_counter(CPUTimeGroups::CPUTimeType name, jlong total);
+
+  static void publish_gc_total_cpu_time();
+};
+
+// Class to compute the total CPU time for a set of threads, then update an
+// hsperfdata counter.
+class ThreadTotalCPUTimeClosure: public ThreadClosure {
+ private:
+  jlong _total;
+  CPUTimeGroups::CPUTimeType _name;
+
+ public:
+  ThreadTotalCPUTimeClosure(CPUTimeGroups::CPUTimeType name)
+      : _total(0), _name(name) {
+    assert(os::is_thread_cpu_time_supported(), "os must support cpu time");
+  }
+
+  ~ThreadTotalCPUTimeClosure();
+
+  virtual void do_thread(Thread* thread);
+};
+
+#endif // SHARE_RUNTIME_CPUTIMECOUNTERS_HPP

src/hotspot/share/runtime/perfData.cpp

@@ -73,6 +73,9 @@ const char* PerfDataManager::_name_spaces[] = {
   "java.threads",           // Threads System name spaces
   "com.sun.threads",
   "sun.threads",
+  "java.threads.cpu_time", //Thread CPU time name spaces
+  "com.sun.threads.cpu_time",
+  "sun.threads.cpu_time",
   "java.property",          // Java Property name spaces
   "com.sun.property",
   "sun.property",

src/hotspot/share/runtime/perfData.hpp

@@ -61,6 +61,9 @@ enum CounterNS {
   JAVA_THREADS,         // Threads System name spaces
   COM_THREADS,
   SUN_THREADS,
+  JAVA_THREADS_CPUTIME, // Thread CPU time name spaces
+  COM_THREADS_CPUTIME,
+  SUN_THREADS_CPUTIME,
   JAVA_PROPERTY,        // Java Property name spaces
   COM_PROPERTY,
   SUN_PROPERTY,

src/hotspot/share/runtime/vmThread.cpp

@@ -35,6 +35,7 @@
 #include "oops/oop.inline.hpp"
 #include "oops/verifyOopClosure.hpp"
 #include "runtime/atomic.hpp"
+#include "runtime/cpuTimeCounters.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/interfaceSupport.inline.hpp"
 #include "runtime/javaThread.inline.hpp"
@@ -136,6 +137,7 @@ void VMThread::create() {
     _perf_accumulated_vm_operation_time =
                  PerfDataManager::create_counter(SUN_THREADS, "vmOperationTime",
                                                  PerfData::U_Ticks, CHECK);
+    CPUTimeCounters::create_counter(CPUTimeGroups::CPUTimeType::vm);
   }
 }
 
@@ -288,6 +290,12 @@ void VMThread::evaluate_operation(VM_Operation* op) {
                      op->evaluate_at_safepoint() ? 0 : 1);
   }
 
+  if (UsePerfData && os::is_thread_cpu_time_supported()) {
+    assert(Thread::current() == this, "Must be called from VM thread");
+    // Update vm_thread_cpu_time after each VM operation.
+    ThreadTotalCPUTimeClosure tttc(CPUTimeGroups::CPUTimeType::vm);
+    tttc.do_thread(this);
+  }
 }
 
 class HandshakeALotClosure : public HandshakeClosure {

test/jdk/sun/tools/jcmd/TestGcCounters.java

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2023 Alphabet LLC.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8315149
+ * @summary Unit test to ensure CPU hsperf counters are published.
+ * @requires vm.gc.G1
+ *
+ * @library /test/lib
+ *
+ * @run main/othervm -XX:+UsePerfData -XX:+UseStringDeduplication TestGcCounters
+ */
+
+import static jdk.test.lib.Asserts.*;
+
+import jdk.test.lib.process.OutputAnalyzer;
+
+public class TestGcCounters {
+
+    private static final String SUN_THREADS = "sun.threads";
+    private static final String SUN_THREADS_CPUTIME = "sun.threads.cpu_time";
+
+    public static void main(String[] args) throws Exception {
+        testGcCpuCountersExist();
+    }
+
+
+    /**
+     * jcmd -J-XX:+UsePerfData pid PerfCounter.print
+     */
+     private static void testGcCpuCountersExist() throws Exception {
+        OutputAnalyzer output = JcmdBase.jcmd(new String[] {"PerfCounter.print"});
+
+        output.shouldHaveExitValue(0);
+        output.shouldContain(SUN_THREADS + ".total_gc_cpu_time");
+        output.shouldContain(SUN_THREADS_CPUTIME + ".gc_conc_mark");
+        output.shouldContain(SUN_THREADS_CPUTIME + ".gc_conc_refine");
+        output.shouldContain(SUN_THREADS_CPUTIME + ".gc_service");
+        output.shouldContain(SUN_THREADS_CPUTIME + ".gc_parallel_workers");
+        output.shouldContain(SUN_THREADS_CPUTIME + ".vm");
+        output.shouldContain(SUN_THREADS_CPUTIME + ".conc_dedup");
+    }
+}
+